Spaces:
Runtime error
Runtime error
Download fusealign_hub/model.py from Zyphra/ZONOS3-Mini: direct link, hf CLI and curl.
- Browser
- Download file 11.9 kB
-
https://huggingface.co/spaces/Zyphra/ZONOS3-Mini/resolve/main/fusealign_hub/model.py
- Command line
-
hf download hf://spaces/Zyphra/ZONOS3-Mini/fusealign_hub/model.py
-
curl -L -o model.py https://huggingface.co/spaces/Zyphra/ZONOS3-Mini/resolve/main/fusealign_hub/model.py
11.9 kB
| """FuseAlign network (inference only). | |
| Parameter names match the training code's state dicts, so exports load with ``strict=True``. | |
| Deviation from the training code: ``RefinementHead`` masks each sample's padded frames, so a padded | |
| batch scores every clip as it would alone. | |
| """ | |
| from dataclasses import dataclass | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| class RMSNorm(nn.Module): | |
| def __init__(self, dim: int, eps: float = 1e-6): | |
| super().__init__() | |
| self.weight = nn.Parameter(torch.ones(dim)) | |
| self.eps = eps | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| normalized = x.float() | |
| normalized = normalized * torch.rsqrt( | |
| normalized.square().mean(dim=-1, keepdim=True) + self.eps | |
| ) | |
| return normalized.to(x.dtype) * self.weight.to(x.dtype) | |
| class RotaryEmbedding(nn.Module): | |
| """RoPE with real-valued positions, shared across the joint blocks.""" | |
| def __init__(self, head_dim: int, theta: float): | |
| super().__init__() | |
| inv_freq = theta ** (-torch.arange(0, head_dim, 2, dtype=torch.float32) / head_dim) | |
| self.register_buffer("inv_freq", inv_freq, persistent=False) | |
| def forward(self, positions: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: | |
| # Keep fractional text positions and their phases in fp32 under autocast. | |
| with torch.autocast(device_type=positions.device.type, enabled=False): | |
| angles = positions.float().unsqueeze(-1) * self.inv_freq.float() | |
| angles = torch.cat((angles, angles), dim=-1).unsqueeze(1) | |
| return angles.cos(), angles.sin() | |
| def _apply_rope( | |
| x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor | |
| ) -> torch.Tensor: | |
| first, second = x.chunk(2, dim=-1) | |
| rotated = torch.cat((-second, first), dim=-1) | |
| return x * cos + rotated * sin | |
| class JointAttention(nn.Module): | |
| def __init__(self, dim: int, n_heads: int): | |
| super().__init__() | |
| self.n_heads = n_heads | |
| self.head_dim = dim // n_heads | |
| self.qkv = nn.Linear(dim, 3 * dim, bias=False) | |
| self.out = nn.Linear(dim, dim, bias=False) | |
| def forward( | |
| self, | |
| x: torch.Tensor, | |
| key_mask: torch.Tensor, | |
| cos: torch.Tensor, | |
| sin: torch.Tensor, | |
| ) -> torch.Tensor: | |
| batch_size, sequence_length, dim = x.shape | |
| qkv = self.qkv(x).reshape( | |
| batch_size, sequence_length, 3, self.n_heads, self.head_dim | |
| ) | |
| q, k, v = qkv.permute(2, 0, 3, 1, 4).unbind(0) | |
| cos, sin = cos.to(q.dtype), sin.to(q.dtype) | |
| q = _apply_rope(q, cos, sin) | |
| k = _apply_rope(k, cos, sin) | |
| # True means an allowed key. Padded queries still see valid keys, so | |
| # no query has an all-masked softmax; blocks zero their outputs below. | |
| attended = F.scaled_dot_product_attention(q, k, v, attn_mask=key_mask) | |
| attended = attended.transpose(1, 2).reshape(batch_size, sequence_length, dim) | |
| return self.out(attended) | |
| class SwiGLU(nn.Module): | |
| def __init__(self, dim: int, ffn_dim: int): | |
| super().__init__() | |
| self.w1 = nn.Linear(dim, ffn_dim, bias=False) | |
| self.w3 = nn.Linear(dim, ffn_dim, bias=False) | |
| self.w2 = nn.Linear(ffn_dim, dim, bias=False) | |
| def forward(self, x: torch.Tensor) -> torch.Tensor: | |
| return self.w2(F.silu(self.w1(x)) * self.w3(x)) | |
| class JointBlock(nn.Module): | |
| def __init__(self, dim: int, n_heads: int, ffn_dim: int): | |
| super().__init__() | |
| self.attn_norm = RMSNorm(dim) | |
| self.attn = JointAttention(dim, n_heads) | |
| self.ffn_norm = RMSNorm(dim) | |
| self.ffn = SwiGLU(dim, ffn_dim) | |
| def forward( | |
| self, | |
| x: torch.Tensor, | |
| key_mask: torch.Tensor, | |
| padding_mask: torch.Tensor, | |
| cos: torch.Tensor, | |
| sin: torch.Tensor, | |
| ) -> torch.Tensor: | |
| x = x + self.attn(self.attn_norm(x), key_mask, cos, sin) | |
| x = x + self.ffn(self.ffn_norm(x)) | |
| return x.masked_fill(padding_mask, 0) | |
| class RefinementHead(nn.Sequential): | |
| def __init__(self, dim: int, feat_dim: int, n_upsample: int): | |
| layers = [nn.Conv1d(dim, feat_dim, kernel_size=1)] | |
| for _ in range(n_upsample): | |
| layers.extend( | |
| [ | |
| nn.ConvTranspose1d( | |
| feat_dim, feat_dim, kernel_size=4, stride=2, padding=1 | |
| ), | |
| nn.GELU(), | |
| ] | |
| ) | |
| layers.extend( | |
| [ | |
| nn.Conv1d(feat_dim, feat_dim, kernel_size=5, padding=2), | |
| nn.GELU(), | |
| nn.Conv1d(feat_dim, feat_dim, kernel_size=1), | |
| ] | |
| ) | |
| super().__init__(*layers) | |
| def forward(self, x: torch.Tensor, lengths: torch.Tensor) -> torch.Tensor: | |
| """Zero each sample's frames past ``lengths`` after every convolution. | |
| Unlike the plain ``nn.Sequential`` used in training: without the masking, conv bias in a | |
| padded batch's padded frames reaches the last valid frames of shorter samples through the | |
| upsampling and kernel-5 convolutions, so a clip's end-of-clip logits would depend on its | |
| batch companions. A no-op for an unpadded batch. | |
| """ | |
| for layer in self: | |
| x = layer(x) | |
| if isinstance(layer, nn.ConvTranspose1d): | |
| lengths = lengths * 2 | |
| if not isinstance(layer, nn.GELU): | |
| frames = torch.arange(x.shape[-1], device=x.device) | |
| x = x.masked_fill(frames[None, None, :] >= lengths[:, None, None], 0) | |
| return x | |
| def _word_mlp(dim: int, output_dim: int) -> nn.Sequential: | |
| return nn.Sequential(nn.Linear(dim, dim), nn.GELU(), nn.Linear(dim, output_dim)) | |
| class FuseAlign(nn.Module): | |
| def __init__(self, args): | |
| super().__init__() | |
| self.args = args | |
| self.audio_patch = nn.Conv1d( | |
| 1, args.dim, kernel_size=args.patch_window, stride=args.patch_hop | |
| ) | |
| self.text_proj = nn.Linear(args.text_dim, args.dim) | |
| self.mod_emb = nn.Parameter(torch.empty(2, args.dim)) | |
| nn.init.normal_(self.mod_emb, mean=0.0, std=0.02) | |
| self.rope = RotaryEmbedding(args.dim // args.n_heads, args.rope_theta) | |
| self.blocks = nn.ModuleList( | |
| JointBlock(args.dim, args.n_heads, args.ffn_dim) | |
| for _ in range(args.n_layers) | |
| ) | |
| self.final_norm = RMSNorm(args.dim) | |
| self.start_refinement = RefinementHead(args.dim, args.feat_dim, args.n_upsample) | |
| self.end_refinement = RefinementHead(args.dim, args.feat_dim, args.n_upsample) | |
| self.start_mlp = _word_mlp(args.dim, args.feat_dim) | |
| self.end_mlp = _word_mlp(args.dim, args.feat_dim) | |
| self.presence_mlp = _word_mlp(args.dim, 1) | |
| nn.init.constant_(self.presence_mlp[-1].bias, args.presence_bias_init) | |
| def _standardize(wav: torch.Tensor, lengths: torch.Tensor) -> torch.Tensor: | |
| sample_mask = torch.arange(wav.shape[1], device=wav.device)[None, :] < lengths[:, None] | |
| with torch.autocast(device_type=wav.device.type, enabled=False): | |
| samples = wav.float().masked_fill(~sample_mask, 0) | |
| counts = lengths[:, None].float() | |
| mean = samples.sum(dim=1, keepdim=True) / counts | |
| centered = (samples - mean).masked_fill(~sample_mask, 0) | |
| std = (centered.square().sum(dim=1, keepdim=True) / counts).sqrt() | |
| return centered / (std + 1e-5) | |
| def forward( | |
| self, | |
| wav: torch.Tensor, | |
| wav_lengths: torch.Tensor, | |
| text_feats: torch.Tensor, | |
| text_mask: torch.Tensor, | |
| word_spans: torch.Tensor, | |
| word_mask: torch.Tensor, | |
| ) -> dict[str, torch.Tensor]: | |
| audio = self.audio_patch(self._standardize(wav, wav_lengths).unsqueeze(1)) | |
| audio = audio.transpose(1, 2) | |
| audio_lengths = (wav_lengths - self.args.patch_window) // self.args.patch_hop + 1 | |
| audio_positions = torch.arange(audio.shape[1], device=wav.device) | |
| audio_mask = audio_positions[None, :] < audio_lengths[:, None] | |
| text = self.text_proj(text_feats) | |
| byte_mask = text_mask[:, 2:].bool() | |
| byte_positions = torch.arange(text.shape[1], device=wav.device) | |
| text_lengths = byte_mask.sum(dim=1) | |
| text_scale = audio_lengths.float() / (text_lengths - 1).clamp_min(1).float() | |
| text_positions = byte_positions[None, :].float() * text_scale[:, None] | |
| text_positions = text_positions.masked_fill(~byte_mask, 0) | |
| audio_rope_positions = audio_positions[None, :].expand(wav.shape[0], -1).float() | |
| audio_rope_positions = audio_rope_positions.masked_fill(~audio_mask, 0) | |
| positions = torch.cat((audio_rope_positions, text_positions), dim=1) | |
| cos, sin = self.rope(positions) | |
| cos, sin = cos.to(audio.dtype), sin.to(audio.dtype) | |
| # Rectangular modality padding does not change the valid joint sequence: | |
| # attention masks out those keys, and RoPE uses per-sample positions. | |
| joint = torch.cat((audio + self.mod_emb[0], text + self.mod_emb[1]), dim=1) | |
| valid_mask = torch.cat((audio_mask, byte_mask), dim=1) | |
| padding_mask = ~valid_mask.unsqueeze(-1) | |
| joint = joint.masked_fill(padding_mask, 0) | |
| key_mask = valid_mask[:, None, None, :] | |
| for block in self.blocks: | |
| joint = block(joint, key_mask, padding_mask, cos, sin) | |
| joint = self.final_norm(joint).masked_fill(padding_mask, 0) | |
| audio_out = joint[:, :audio.shape[1]].transpose(1, 2) | |
| text_out = joint[:, audio.shape[1]:] | |
| membership = ( | |
| (byte_positions[None, None, :] >= word_spans[:, :, 0, None]) | |
| & (byte_positions[None, None, :] < word_spans[:, :, 1, None]) | |
| & byte_mask[:, None, :] | |
| & word_mask[:, :, None] | |
| ) | |
| pooling = membership.to(text_out.dtype) | |
| pooling = pooling / pooling.sum(dim=-1, keepdim=True).clamp_min(1) | |
| word_vectors = torch.bmm(pooling, text_out) | |
| start_features = self.start_refinement(audio_out, audio_lengths) | |
| end_features = self.end_refinement(audio_out, audio_lengths) | |
| with torch.autocast(device_type=wav.device.type, enabled=False): | |
| word_vectors = word_vectors.float() | |
| start_logits = torch.einsum( | |
| "bwf,bft->bwt", self.start_mlp(word_vectors), start_features.float() | |
| ) | |
| end_logits = torch.einsum( | |
| "bwf,bft->bwt", self.end_mlp(word_vectors), end_features.float() | |
| ) | |
| presence_logits = self.presence_mlp(word_vectors).squeeze(-1) | |
| return { | |
| "start_logits": start_logits, | |
| "end_logits": end_logits, | |
| "presence_logits": presence_logits, | |
| "frame_lengths": audio_lengths * (2 ** self.args.n_upsample), | |
| } | |
| class FuseAlignArgs: | |
| dim: int = 1024 | |
| n_layers: int = 12 | |
| n_heads: int = 16 | |
| ffn_dim: int = 4096 | |
| feat_dim: int = 256 | |
| patch_window: int = 512 | |
| patch_hop: int = 256 | |
| n_upsample: int = 4 | |
| text_dim: int = 1024 | |
| rope_theta: float = 1e4 | |
| presence_bias_init: float = 2.0 | |
| dia_repo: str = "nari-labs/Dia-1.6B-0626" | |
| def cast_autocast_weights(net: FuseAlign) -> None: | |
| """Store in bf16 the weights CUDA autocast casts to bf16 on every forward. | |
| Bit-identical (autocast rounds the same fp32 values the same way) and drops ~200 cast | |
| kernels per forward. RMSNorm weights, ``mod_emb`` and the word MLPs, which run with | |
| autocast disabled, stay fp32. | |
| """ | |
| fp32 = {id(layer) for mlp in (net.start_mlp, net.end_mlp, net.presence_mlp) for layer in mlp} | |
| for module in net.modules(): | |
| if isinstance(module, (nn.Linear, nn.Conv1d, nn.ConvTranspose1d)) and id(module) not in fp32: | |
| module.to(torch.bfloat16) | |