Gatozu35's picture
Deploy ZONOS3 212cb76-dirty
d963eb6 verified
Raw History Blame Contribute Delete
11.9 kB
"""FuseAlign network (inference only).
Parameter names match the training code's state dicts, so exports load with ``strict=True``.
Deviation from the training code: ``RefinementHead`` masks each sample's padded frames, so a padded
batch scores every clip as it would alone.
"""
from dataclasses import dataclass
import torch
from torch import nn
from torch.nn import functional as F
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.weight = nn.Parameter(torch.ones(dim))
self.eps = eps
def forward(self, x: torch.Tensor) -> torch.Tensor:
normalized = x.float()
normalized = normalized * torch.rsqrt(
normalized.square().mean(dim=-1, keepdim=True) + self.eps
)
return normalized.to(x.dtype) * self.weight.to(x.dtype)
class RotaryEmbedding(nn.Module):
"""RoPE with real-valued positions, shared across the joint blocks."""
def __init__(self, head_dim: int, theta: float):
super().__init__()
inv_freq = theta ** (-torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim)
self.register_buffer("inv_freq", inv_freq, persistent=False)
def forward(self, positions: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
# Keep fractional text positions and their phases in fp32 under autocast.
with torch.autocast(device_type=positions.device.type, enabled=False):
angles = positions.float().unsqueeze(-1) * self.inv_freq.float()
angles = torch.cat((angles, angles), dim=-1).unsqueeze(1)
return angles.cos(), angles.sin()
def _apply_rope(
x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor
) -> torch.Tensor:
first, second = x.chunk(2, dim=-1)
rotated = torch.cat((-second, first), dim=-1)
return x * cos + rotated * sin
class JointAttention(nn.Module):
def __init__(self, dim: int, n_heads: int):
super().__init__()
self.n_heads = n_heads
self.head_dim = dim // n_heads
self.qkv = nn.Linear(dim, 3 * dim, bias=False)
self.out = nn.Linear(dim, dim, bias=False)
def forward(
self,
x: torch.Tensor,
key_mask: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
) -> torch.Tensor:
batch_size, sequence_length, dim = x.shape
qkv = self.qkv(x).reshape(
batch_size, sequence_length, 3, self.n_heads, self.head_dim
)
q, k, v = qkv.permute(2, 0, 3, 1, 4).unbind(0)
cos, sin = cos.to(q.dtype), sin.to(q.dtype)
q = _apply_rope(q, cos, sin)
k = _apply_rope(k, cos, sin)
# True means an allowed key. Padded queries still see valid keys, so
# no query has an all-masked softmax; blocks zero their outputs below.
attended = F.scaled_dot_product_attention(q, k, v, attn_mask=key_mask)
attended = attended.transpose(1, 2).reshape(batch_size, sequence_length, dim)
return self.out(attended)
class SwiGLU(nn.Module):
def __init__(self, dim: int, ffn_dim: int):
super().__init__()
self.w1 = nn.Linear(dim, ffn_dim, bias=False)
self.w3 = nn.Linear(dim, ffn_dim, bias=False)
self.w2 = nn.Linear(ffn_dim, dim, bias=False)
def forward(self, x: torch.Tensor) -> torch.Tensor:
return self.w2(F.silu(self.w1(x)) * self.w3(x))
class JointBlock(nn.Module):
def __init__(self, dim: int, n_heads: int, ffn_dim: int):
super().__init__()
self.attn_norm = RMSNorm(dim)
self.attn = JointAttention(dim, n_heads)
self.ffn_norm = RMSNorm(dim)
self.ffn = SwiGLU(dim, ffn_dim)
def forward(
self,
x: torch.Tensor,
key_mask: torch.Tensor,
padding_mask: torch.Tensor,
cos: torch.Tensor,
sin: torch.Tensor,
) -> torch.Tensor:
x = x + self.attn(self.attn_norm(x), key_mask, cos, sin)
x = x + self.ffn(self.ffn_norm(x))
return x.masked_fill(padding_mask, 0)
class RefinementHead(nn.Sequential):
def __init__(self, dim: int, feat_dim: int, n_upsample: int):
layers = [nn.Conv1d(dim, feat_dim, kernel_size=1)]
for _ in range(n_upsample):
layers.extend(
[
nn.ConvTranspose1d(
feat_dim, feat_dim, kernel_size=4, stride=2, padding=1
),
nn.GELU(),
]
)
layers.extend(
[
nn.Conv1d(feat_dim, feat_dim, kernel_size=5, padding=2),
nn.GELU(),
nn.Conv1d(feat_dim, feat_dim, kernel_size=1),
]
)
super().__init__(*layers)
def forward(self, x: torch.Tensor, lengths: torch.Tensor) -> torch.Tensor:
"""Zero each sample's frames past ``lengths`` after every convolution.
Unlike the plain ``nn.Sequential`` used in training: without the masking, conv bias in a
padded batch's padded frames reaches the last valid frames of shorter samples through the
upsampling and kernel-5 convolutions, so a clip's end-of-clip logits would depend on its
batch companions. A no-op for an unpadded batch.
"""
for layer in self:
x = layer(x)
if isinstance(layer, nn.ConvTranspose1d):
lengths = lengths * 2
if not isinstance(layer, nn.GELU):
frames = torch.arange(x.shape[-1], device=x.device)
x = x.masked_fill(frames[None, None, :] >= lengths[:, None, None], 0)
return x
def _word_mlp(dim: int, output_dim: int) -> nn.Sequential:
return nn.Sequential(nn.Linear(dim, dim), nn.GELU(), nn.Linear(dim, output_dim))
class FuseAlign(nn.Module):
def __init__(self, args):
super().__init__()
self.args = args
self.audio_patch = nn.Conv1d(
1, args.dim, kernel_size=args.patch_window, stride=args.patch_hop
)
self.text_proj = nn.Linear(args.text_dim, args.dim)
self.mod_emb = nn.Parameter(torch.empty(2, args.dim))
nn.init.normal_(self.mod_emb, mean=0.0, std=0.02)
self.rope = RotaryEmbedding(args.dim // args.n_heads, args.rope_theta)
self.blocks = nn.ModuleList(
JointBlock(args.dim, args.n_heads, args.ffn_dim)
for _ in range(args.n_layers)
)
self.final_norm = RMSNorm(args.dim)
self.start_refinement = RefinementHead(args.dim, args.feat_dim, args.n_upsample)
self.end_refinement = RefinementHead(args.dim, args.feat_dim, args.n_upsample)
self.start_mlp = _word_mlp(args.dim, args.feat_dim)
self.end_mlp = _word_mlp(args.dim, args.feat_dim)
self.presence_mlp = _word_mlp(args.dim, 1)
nn.init.constant_(self.presence_mlp[-1].bias, args.presence_bias_init)
@staticmethod
def _standardize(wav: torch.Tensor, lengths: torch.Tensor) -> torch.Tensor:
sample_mask = torch.arange(wav.shape[1], device=wav.device)[None, :] < lengths[:, None]
with torch.autocast(device_type=wav.device.type, enabled=False):
samples = wav.float().masked_fill(~sample_mask, 0)
counts = lengths[:, None].float()
mean = samples.sum(dim=1, keepdim=True) / counts
centered = (samples - mean).masked_fill(~sample_mask, 0)
std = (centered.square().sum(dim=1, keepdim=True) / counts).sqrt()
return centered / (std + 1e-5)
def forward(
self,
wav: torch.Tensor,
wav_lengths: torch.Tensor,
text_feats: torch.Tensor,
text_mask: torch.Tensor,
word_spans: torch.Tensor,
word_mask: torch.Tensor,
) -> dict[str, torch.Tensor]:
audio = self.audio_patch(self._standardize(wav, wav_lengths).unsqueeze(1))
audio = audio.transpose(1, 2)
audio_lengths = (wav_lengths - self.args.patch_window) // self.args.patch_hop + 1
audio_positions = torch.arange(audio.shape[1], device=wav.device)
audio_mask = audio_positions[None, :] < audio_lengths[:, None]
text = self.text_proj(text_feats)
byte_mask = text_mask[:, 2:].bool()
byte_positions = torch.arange(text.shape[1], device=wav.device)
text_lengths = byte_mask.sum(dim=1)
text_scale = audio_lengths.float() / (text_lengths - 1).clamp_min(1).float()
text_positions = byte_positions[None, :].float() * text_scale[:, None]
text_positions = text_positions.masked_fill(~byte_mask, 0)
audio_rope_positions = audio_positions[None, :].expand(wav.shape[0], -1).float()
audio_rope_positions = audio_rope_positions.masked_fill(~audio_mask, 0)
positions = torch.cat((audio_rope_positions, text_positions), dim=1)
cos, sin = self.rope(positions)
cos, sin = cos.to(audio.dtype), sin.to(audio.dtype)
# Rectangular modality padding does not change the valid joint sequence:
# attention masks out those keys, and RoPE uses per-sample positions.
joint = torch.cat((audio + self.mod_emb[0], text + self.mod_emb[1]), dim=1)
valid_mask = torch.cat((audio_mask, byte_mask), dim=1)
padding_mask = ~valid_mask.unsqueeze(-1)
joint = joint.masked_fill(padding_mask, 0)
key_mask = valid_mask[:, None, None, :]
for block in self.blocks:
joint = block(joint, key_mask, padding_mask, cos, sin)
joint = self.final_norm(joint).masked_fill(padding_mask, 0)
audio_out = joint[:, :audio.shape[1]].transpose(1, 2)
text_out = joint[:, audio.shape[1]:]
membership = (
(byte_positions[None, None, :] >= word_spans[:, :, 0, None])
& (byte_positions[None, None, :] < word_spans[:, :, 1, None])
& byte_mask[:, None, :]
& word_mask[:, :, None]
)
pooling = membership.to(text_out.dtype)
pooling = pooling / pooling.sum(dim=-1, keepdim=True).clamp_min(1)
word_vectors = torch.bmm(pooling, text_out)
start_features = self.start_refinement(audio_out, audio_lengths)
end_features = self.end_refinement(audio_out, audio_lengths)
with torch.autocast(device_type=wav.device.type, enabled=False):
word_vectors = word_vectors.float()
start_logits = torch.einsum(
"bwf,bft->bwt", self.start_mlp(word_vectors), start_features.float()
)
end_logits = torch.einsum(
"bwf,bft->bwt", self.end_mlp(word_vectors), end_features.float()
)
presence_logits = self.presence_mlp(word_vectors).squeeze(-1)
return {
"start_logits": start_logits,
"end_logits": end_logits,
"presence_logits": presence_logits,
"frame_lengths": audio_lengths * (2 ** self.args.n_upsample),
}
@dataclass
class FuseAlignArgs:
dim: int = 1024
n_layers: int = 12
n_heads: int = 16
ffn_dim: int = 4096
feat_dim: int = 256
patch_window: int = 512
patch_hop: int = 256
n_upsample: int = 4
text_dim: int = 1024
rope_theta: float = 1e4
presence_bias_init: float = 2.0
dia_repo: str = "nari-labs/Dia-1.6B-0626"
def cast_autocast_weights(net: FuseAlign) -> None:
"""Store in bf16 the weights CUDA autocast casts to bf16 on every forward.
Bit-identical (autocast rounds the same fp32 values the same way) and drops ~200 cast
kernels per forward. RMSNorm weights, ``mod_emb`` and the word MLPs, which run with
autocast disabled, stay fp32.
"""
fp32 = {id(layer) for mlp in (net.start_mlp, net.end_mlp, net.presence_mlp) for layer in mlp}
for module in net.modules():
if isinstance(module, (nn.Linear, nn.Conv1d, nn.ConvTranspose1d)) and id(module) not in fp32:
module.to(torch.bfloat16)