s3-s2st / s3_encoder.py
multimodalart's picture
multimodalart HF Staff
Initial S3-S2ST demo
1829c79 verified
Raw History Blame Contribute Delete
39.1 kB
# Audio encoder for S3-S2ST, extracted from the authors' vLLM fork # (S3-S2ST/vendor/vllm/model_executor/models/omnidrca2_encoder.py, Apache-2.0, # adapted from FunAudioLLM/Fun-Audio-Chat). vLLM-specific imports removed. import logging import math import os from typing import Optional, Tuple import numpy as np import torch import torch.nn.functional as F from torch import Tensor, nn logger = logging.getLogger(__name__) def make_pad_mask(lengths, xs=None, length_dim=-1, maxlen=None): """Make mask tensor containing indices of padded part. Args: lengths (LongTensor or List): Batch of lengths (B,). xs (Tensor, optional): The reference tensor. If set, masks will be the same shape as this tensor. length_dim (int, optional): Dimension indicator of the above tensor. See the example. Returns: Tensor: Mask tensor containing indices of padded part. dtype=torch.uint8 in PyTorch 1.2- dtype=torch.bool in PyTorch 1.2+ (including 1.2) Examples: With only lengths. >>> lengths = [5, 3, 2] >>> make_pad_mask(lengths) masks = [[0, 0, 0, 0 ,0], [0, 0, 0, 1, 1], [0, 0, 1, 1, 1]] With the reference tensor. >>> xs = torch.zeros((3, 2, 4)) >>> make_pad_mask(lengths, xs) tensor([[[0, 0, 0, 0], [0, 0, 0, 0]], [[0, 0, 0, 1], [0, 0, 0, 1]], [[0, 0, 1, 1], [0, 0, 1, 1]]], dtype=torch.uint8) >>> xs = torch.zeros((3, 2, 6)) >>> make_pad_mask(lengths, xs) tensor([[[0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 0, 1]], [[0, 0, 0, 1, 1, 1], [0, 0, 0, 1, 1, 1]], [[0, 0, 1, 1, 1, 1], [0, 0, 1, 1, 1, 1]]], dtype=torch.uint8) With the reference tensor and dimension indicator. >>> xs = torch.zeros((3, 6, 6)) >>> make_pad_mask(lengths, xs, 1) tensor([[[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1]], [[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1]], [[0, 0, 0, 0, 0, 0], [0, 0, 0, 0, 0, 0], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1], [1, 1, 1, 1, 1, 1]]], dtype=torch.uint8) >>> make_pad_mask(lengths, xs, 2) tensor([[[0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 0, 1], [0, 0, 0, 0, 0, 1]], [[0, 0, 0, 1, 1, 1], [0, 0, 0, 1, 1, 1], [0, 0, 0, 1, 1, 1], [0, 0, 0, 1, 1, 1], [0, 0, 0, 1, 1, 1], [0, 0, 0, 1, 1, 1]], [[0, 0, 1, 1, 1, 1], [0, 0, 1, 1, 1, 1], [0, 0, 1, 1, 1, 1], [0, 0, 1, 1, 1, 1], [0, 0, 1, 1, 1, 1], [0, 0, 1, 1, 1, 1]]], dtype=torch.uint8) """ if length_dim == 0: raise ValueError("length_dim cannot be 0: {}".format(length_dim)) if not isinstance(lengths, list): lengths = lengths.tolist() bs = int(len(lengths)) if maxlen is None: if xs is None: maxlen = int(max(lengths)) else: maxlen = xs.size(length_dim) else: assert xs is None assert maxlen >= int(max(lengths)) seq_range = torch.arange(0, maxlen, dtype=torch.int64) seq_range_expand = seq_range.unsqueeze(0).expand(bs, maxlen) seq_length_expand = seq_range_expand.new(lengths).unsqueeze(-1) mask = seq_range_expand >= seq_length_expand if xs is not None: assert xs.size(0) == bs, (xs.size(0), bs) if length_dim < 0: length_dim = xs.dim() + length_dim # ind = (:, None, ..., None, :, , None, ..., None) ind = tuple(slice(None) if i in (0, length_dim) else None for i in range(xs.dim())) mask = mask[ind].expand_as(xs).to(xs.device) return mask class LayerNorm(nn.LayerNorm): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) def forward(self, input): output = F.layer_norm( input.float(), self.normalized_shape, self.weight.float() if self.weight is not None else None, self.bias.float() if self.bias is not None else None, self.eps, ) return output.type_as(input) # # encoder: SenseVoiceEncoder # encoder_conf = { # "input_size": 128, # "attention_heads": 20, # "linear_units": 1280, # "num_blocks": 32, # "dropout_rate": 0.1, # "positional_dropout_rate": 0.1, # "attention_dropout_rate": 0.1, # "kernel_size": 31, # "sanm_shfit": 0, # "att_type": "self_att_fsmn_sdpa", # "downsample_rate": 4, # "use_padmask": True, # "max_position_embeddings": 2048, # "rope_theta": 10000, # "freeze": False, # "freeze_exclude_key": "none", # } # Copied from transformers.models.llama.modeling_llama.rotate_half def rotate_half(x): """Rotates half the hidden dims of the input.""" x1 = x[..., : x.shape[-1] // 2] x2 = x[..., x.shape[-1] // 2 :] return torch.cat((-x2, x1), dim=-1) # Copied from transformers.models.mistral.modeling_mistral.apply_rotary_pos_emb def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1): """Applies Rotary Position Embedding to the query and key tensors. Args: q (`torch.Tensor`): The query tensor. k (`torch.Tensor`): The key tensor. cos (`torch.Tensor`): The cosine part of the rotary embedding. sin (`torch.Tensor`): The sine part of the rotary embedding. position_ids (`torch.Tensor`): The position indices of the tokens corresponding to the query and key tensors. For example, this can be used to pass offsetted position ids when working with a KV-cache. unsqueeze_dim (`int`, *optional*, defaults to 1): The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2. Returns: `tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding. """ cos = cos[position_ids].unsqueeze(unsqueeze_dim) sin = sin[position_ids].unsqueeze(unsqueeze_dim) q_embed = (q * cos) + (rotate_half(q) * sin) k_embed = (k * cos) + (rotate_half(k) * sin) return q_embed, k_embed class RotaryEmbedding(nn.Module): def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None): super().__init__() self.dim = dim self.max_position_embeddings = max_position_embeddings self.base = base inv_freq = 1.0 / ( self.base ** (torch.arange(0, self.dim, 2, dtype=torch.int64).float().to(device) / self.dim) ) self.register_buffer("inv_freq", inv_freq, persistent=False) # Build here to make `torch.jit.trace` work. self._set_cos_sin_cache( seq_len=max_position_embeddings, device=self.inv_freq.device, dtype=torch.get_default_dtype(), ) def _set_cos_sin_cache(self, seq_len, device, dtype): self.max_seq_len_cached = seq_len t = torch.arange(self.max_seq_len_cached, device=device, dtype=torch.int64).type_as( self.inv_freq ) freqs = torch.outer(t, self.inv_freq) # Different from paper, but it uses a different permutation in order to obtain the same calculation emb = torch.cat((freqs, freqs), dim=-1) self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False) self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False) def forward(self, x, seq_len=None): # x: [bs, num_attention_heads, seq_len, head_size] if seq_len > self.max_seq_len_cached: self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype) return ( self.cos_cached[:seq_len].to(dtype=x.dtype), self.sin_cached[:seq_len].to(dtype=x.dtype), ) class Linear(nn.Linear): def forward(self, x: Tensor) -> Tensor: return F.linear( x, self.weight.to(x.dtype), None if self.bias is None else self.bias.to(x.dtype), ) class MultiHeadAttention(nn.Module): def __init__(self, n_state: int, n_head: int): super().__init__() self.n_head = n_head self.query = Linear(n_state, n_state) self.key = Linear(n_state, n_state, bias=False) self.value = Linear(n_state, n_state) self.out = Linear(n_state, n_state) def forward( self, x: Tensor, xa: Optional[Tensor] = None, mask: Optional[Tensor] = None, kv_cache: Optional[dict] = None, **kwargs, ): is_pad_mask = kwargs.get("is_pad_mask", False) q = self.query(x) if kv_cache is None or xa is None or self.key not in kv_cache: # hooks, if installed (i.e. kv_cache is not None), will prepend the cached kv tensors; # otherwise, perform key/value projections for self- or cross-attention as usual. k = self.key(x if xa is None else xa) v = self.value(x if xa is None else xa) else: # for cross-attention, calculate keys and values once and reuse in subsequent calls. k = kv_cache[self.key] v = kv_cache[self.value] wv, qk = self.qkv_attention(q, k, v, mask, is_pad_mask=is_pad_mask) return self.out(wv), qk def qkv_attention( self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None, **kwargs, ): is_pad_mask = kwargs.get("is_pad_mask", False) n_batch, n_ctx, n_state = q.shape scale = (n_state // self.n_head) ** -0.25 q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) * scale k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 3, 1) * scale v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) qk = q @ k if mask is not None: if not is_pad_mask: qk = qk + mask[:n_ctx, :n_ctx] else: mask = mask.unsqueeze(1).eq(0) # (batch, 1, t, 1) min_value = -float( "inf" ) # min_value = float(np.finfo(torch.tensor(0, dtype=qk.dtype).numpy().dtype).min) qk = qk.masked_fill(mask, min_value) qk = qk.float() w = F.softmax(qk, dim=-1).to(q.dtype) if mask is not None and is_pad_mask: w = w.masked_fill(mask, 0.0) return (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2), qk.detach() class MultiHeadAttentionSdpa(nn.Module): def __init__(self, n_state: int, n_head: int): super().__init__() self.n_head = n_head self.query = Linear(n_state, n_state) self.key = Linear(n_state, n_state, bias=False) self.value = Linear(n_state, n_state) self.out = Linear(n_state, n_state) def forward( self, x: Tensor, xa: Optional[Tensor] = None, mask: Optional[Tensor] = None, kv_cache: Optional[dict] = None, **kwargs, ): is_pad_mask = kwargs.get("is_pad_mask", False) q = self.query(x) if kv_cache is None or xa is None or self.key not in kv_cache: # hooks, if installed (i.e. kv_cache is not None), will prepend the cached kv tensors; # otherwise, perform key/value projections for self- or cross-attention as usual. k = self.key(x if xa is None else xa) v = self.value(x if xa is None else xa) else: # for cross-attention, calculate keys and values once and reuse in subsequent calls. k = kv_cache[self.key] v = kv_cache[self.value] wv, qk = self.qkv_attention(q, k, v, mask, is_pad_mask=is_pad_mask, is_causal=False) return self.out(wv), qk def qkv_attention( self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None, **kwargs, ): is_pad_mask = kwargs.get("is_pad_mask", False) is_causal = kwargs.get("is_causal", False) n_batch, n_ctx, n_state = q.shape scale = (n_state // self.n_head) ** -0.5 q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) if mask is not None: if not is_pad_mask: mask = None is_causal = True else: mask = mask.unsqueeze(1).to(torch.bool) # (batch, 1, 1, t) attn_output = torch.nn.functional.scaled_dot_product_attention( q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=is_causal, scale=scale, ) if mask is not None: attn_output = attn_output.masked_fill(mask.transpose(2, 3).logical_not(), 0.0) attn_output = attn_output.transpose(1, 2) attn_output = attn_output.flatten(start_dim=2) return attn_output, None class MultiHeadAttentionRoPE(nn.Module): def __init__(self, linear_units: int, attention_heads: int, **kwargs): super().__init__() self.attention_heads = attention_heads self.query = Linear(linear_units, linear_units) self.key = Linear(linear_units, linear_units, bias=False) self.value = Linear(linear_units, linear_units) self.out = Linear(linear_units, linear_units) self.rotary_emb = RotaryEmbedding( linear_units // attention_heads, max_position_embeddings=kwargs.get("max_position_embeddings", 2048), base=kwargs.get("rope_theta", 10000), ) def forward( self, x: Tensor, mask: Optional[Tensor] = None, **kwargs, ): q = self.query(x) k = self.key(x) v = self.value(x) wv, qk = self.qkv_attention(q, k, v, mask, **kwargs) return self.out(wv), qk def qkv_attention( self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None, **kwargs, ): n_batch, n_ctx, n_state = q.shape scale = (n_state // self.n_head) ** -0.25 q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) * scale k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 3, 1) * scale v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) position_ids = kwargs.get("position_ids", None) kv_seq_len = v.shape[-2] cos, sin = self.rotary_emb(v, seq_len=kv_seq_len) q, k = apply_rotary_pos_emb(q, k, cos, sin, position_ids) qk = q @ k if mask is not None: mask = mask.unsqueeze(1).eq(0) # (batch, 1, t, 1) min_value = -float( "inf" ) # min_value = float(np.finfo(torch.tensor(0, dtype=qk.dtype).numpy().dtype).min) qk = qk.masked_fill(mask, min_value) qk = qk.float() w = F.softmax(qk, dim=-1).to(q.dtype) if mask is not None: w = w.masked_fill(mask, 0.0) return (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2), qk.detach() class MultiHeadAttentionSdpaRoPE(nn.Module): def __init__(self, linear_units: int, attention_heads: int, **kwargs): super().__init__() self.attention_heads = attention_heads self.query = Linear(linear_units, linear_units) self.key = Linear(linear_units, linear_units, bias=False) self.value = Linear(linear_units, linear_units) self.out = Linear(linear_units, linear_units) self.rotary_emb = RotaryEmbedding( linear_units // attention_heads, max_position_embeddings=kwargs.get("max_position_embeddings", 2048), base=kwargs.get("rope_theta", 10000), ) def forward( self, x: Tensor, mask: Optional[Tensor] = None, **kwargs, ): q = self.query(x) k = self.key(x) v = self.value(x) wv, qk = self.qkv_attention(q, k, v, mask, **kwargs) return self.out(wv), qk def qkv_attention( self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None, **kwargs, ): is_causal = kwargs.get("is_causal", False) n_batch, n_ctx, n_state = q.shape scale = (n_state // self.n_head) ** -0.5 q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) position_ids = kwargs.get("position_ids", None) kv_seq_len = v.shape[-2] cos, sin = self.rotary_emb(v, seq_len=kv_seq_len) q, k = apply_rotary_pos_emb(q, k, cos, sin, position_ids) if mask is not None: mask = mask.unsqueeze(1).to(torch.bool) # (batch, 1, 1, t) attn_output = torch.nn.functional.scaled_dot_product_attention( q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=is_causal, scale=scale, ) if mask is not None: attn_output = attn_output.masked_fill(mask.transpose(2, 3).logical_not(), 0.0) attn_output = attn_output.transpose(1, 2) attn_output = attn_output.flatten(start_dim=2) return attn_output, None class MultiHeadAttentionFSMNRoPE(nn.Module): def __init__(self, linear_units: int, attention_heads: int, **kwargs): super().__init__() self.attention_heads = attention_heads self.query = Linear(linear_units, linear_units) self.key = Linear(linear_units, linear_units, bias=False) self.value = Linear(linear_units, linear_units) self.out = Linear(linear_units, linear_units) self.rotary_emb = RotaryEmbedding( linear_units // attention_heads, max_position_embeddings=kwargs.get("max_position_embeddings", 2048), base=kwargs.get("rope_theta", 10000), ) self.fsmn_block = nn.Conv1d( linear_units, linear_units, kwargs.get("kernel_size", 15), stride=1, padding=0, groups=linear_units, bias=False, ) # padding left_padding = (kwargs.get("kernel_size", 15) - 1) // 2 left_padding = left_padding + kwargs.get("sanm_shfit", 0) right_padding = kwargs.get("kernel_size", 15) - 1 - left_padding self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0) self.dropout = torch.nn.Dropout(kwargs.get("dropout_rate", 0.0)) def fsmn(self, inputs, mask): b, t, d = inputs.size() if mask is not None: mask = torch.reshape(mask, (b, -1, 1)) inputs = inputs * mask x = inputs.transpose(1, 2) x = self.pad_fn(x) x = self.fsmn_block(x) x = x.transpose(1, 2) + inputs x = self.dropout(x) if mask is not None: x = x * mask return x def forward( self, x: Tensor, mask: Optional[Tensor] = None, **kwargs, ): q = self.query(x) k = self.key(x) v = self.value(x) memory = self.fsmn(v, mask=mask) wv, qk = self.qkv_attention(q, k, v, mask, **kwargs) return self.out(wv) + memory, qk def qkv_attention( self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None, **kwargs, ): b, t, d = q.shape scale = (d // self.attention_heads) ** -0.5 q = q.view(*q.shape[:2], self.attention_heads, -1).permute(0, 2, 1, 3) k = k.view(*k.shape[:2], self.attention_heads, -1).permute(0, 2, 1, 3) v = v.view(*v.shape[:2], self.attention_heads, -1).permute(0, 2, 1, 3) position_ids = kwargs.get("position_ids", None) kv_seq_len = v.shape[-2] cos, sin = self.rotary_emb(v, seq_len=kv_seq_len) q, k = apply_rotary_pos_emb(q, k, cos, sin, position_ids) qk = q @ k if mask is not None: mask = mask.unsqueeze(1).eq(0) # (batch, 1, t, 1) min_value = -float( "inf" ) # min_value = float(np.finfo(torch.tensor(0, dtype=qk.dtype).numpy().dtype).min) qk = qk.masked_fill(mask, min_value) qk = qk.float() w = F.softmax(qk, dim=-1).to(q.dtype) if mask is not None: w = w.masked_fill(mask, 0.0) return (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2), qk.detach() class MultiHeadAttentionFSMNSdpaRoPE(nn.Module): def __init__(self, linear_units: int, attention_heads: int, **kwargs): super().__init__() self.attention_heads = attention_heads self.query = Linear(linear_units, linear_units) self.key = Linear(linear_units, linear_units, bias=False) self.value = Linear(linear_units, linear_units) self.out = Linear(linear_units, linear_units) self.rotary_emb = RotaryEmbedding( linear_units // attention_heads, max_position_embeddings=kwargs.get("max_position_embeddings", 2048), base=kwargs.get("rope_theta", 10000), ) self.fsmn_block = nn.Conv1d( linear_units, linear_units, kwargs.get("kernel_size", 15), stride=1, padding=0, groups=linear_units, bias=False, ) # padding left_padding = (kwargs.get("kernel_size", 15) - 1) // 2 left_padding = left_padding + kwargs.get("sanm_shfit", 0) right_padding = kwargs.get("kernel_size", 15) - 1 - left_padding self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0) self.dropout = torch.nn.Dropout(kwargs.get("dropout_rate", 0.0)) def fsmn(self, inputs, mask): b, t, d = inputs.size() # b, t, d if mask is not None: # A streaming attention mask is [B, T_query, T_key]. FSMN only # needs the per-frame validity mask, which is its diagonal. frame_mask = ( torch.diagonal(mask, dim1=-2, dim2=-1) if mask.dim() == 3 else torch.reshape(mask, (b, -1)) ) frame_mask = frame_mask.unsqueeze(-1) # b, t, 1 inputs = inputs * frame_mask x = inputs.transpose(1, 2) x = self.pad_fn(x) x = self.fsmn_block(x) x = x.transpose(1, 2) + inputs x = self.dropout(x) if mask is not None: x = x * frame_mask return x def forward( self, x: Tensor, mask: Optional[Tensor] = None, **kwargs, ): q = self.query(x) k = self.key(x) v = self.value(x) memory = self.fsmn(v, mask=mask) wv, qk = self.qkv_attention(q, k, v, mask, **kwargs) return self.out(wv) + memory, qk def qkv_attention( self, q: Tensor, k: Tensor, v: Tensor, mask: Optional[Tensor] = None, **kwargs, ): is_causal = kwargs.get("is_causal", False) b, t, d = q.shape scale = (d // self.attention_heads) ** -0.5 q = q.view(*q.shape[:2], self.attention_heads, -1).permute(0, 2, 1, 3) k = k.view(*k.shape[:2], self.attention_heads, -1).permute(0, 2, 1, 3) v = v.view(*v.shape[:2], self.attention_heads, -1).permute(0, 2, 1, 3) position_ids = kwargs.get("position_ids", None) kv_seq_len = v.shape[-2] cos, sin = self.rotary_emb(v, seq_len=kv_seq_len) q, k = apply_rotary_pos_emb(q, k, cos, sin, position_ids) query_mask = None if mask is not None: if mask.dim() == 3: query_mask = torch.diagonal(mask, dim1=-2, dim2=-1) else: query_mask = mask.reshape(b, -1) mask = mask.unsqueeze(1).to(torch.bool) # (batch, 1, *, t) attn_output = torch.nn.functional.scaled_dot_product_attention( q, k, v, attn_mask=mask, dropout_p=0.0, is_causal=is_causal, scale=scale, ) if query_mask is not None: attn_output = attn_output.masked_fill( query_mask[:, None, :, None].logical_not(), 0.0 ) attn_output = attn_output.transpose(1, 2) attn_output = attn_output.flatten(start_dim=2) return attn_output, None att_type_dict = { "default": MultiHeadAttention, "sdpa": MultiHeadAttentionSdpa, "self_att": MultiHeadAttentionRoPE, "self_att_sdpa": MultiHeadAttentionSdpaRoPE, "self_att_fsmn": MultiHeadAttentionFSMNRoPE, "self_att_fsmn_sdpa": MultiHeadAttentionFSMNSdpaRoPE, } class EncoderLayerSANMLarge(nn.Module): def __init__(self, linear_units: int, attention_heads: int, **kwargs): super().__init__() att_type = kwargs.get("att_type", "self_att_fsmn_sdpa") self.attn = att_type_dict[att_type](linear_units, attention_heads, **kwargs) self.attn_ln = LayerNorm(linear_units) n_mlp = linear_units * 4 self.mlp = nn.Sequential( Linear(linear_units, n_mlp), nn.GELU(), Linear(n_mlp, linear_units) ) self.mlp_ln = LayerNorm(linear_units) def forward( self, x: Tensor, mask: Optional[Tensor] = None, **kwargs, ): x = x + self.attn(self.attn_ln(x), mask=mask, **kwargs)[0] x = x + self.mlp(self.mlp_ln(x)) return x class Conv1d(nn.Conv1d): def _conv_forward(self, x, weight, bias): return super()._conv_forward( x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype) ) class SenseVoiceEncoder(nn.Module): def __init__( self, input_size, linear_units: int, attention_heads: int, num_blocks: int, **kwargs, ): super().__init__() self.conv1 = Conv1d(input_size, linear_units, kernel_size=3, stride=2, padding=1) self.conv2 = Conv1d(linear_units, linear_units, kernel_size=3, stride=2, padding=1) self.blocks = nn.ModuleList( [ EncoderLayerSANMLarge(linear_units, attention_heads, **kwargs) for _ in range(num_blocks) ] ) self.ln_post = LayerNorm(linear_units) self.use_padmask = kwargs.get("use_padmask", True) self.downsample_rate = kwargs.get("downsample_rate", 4) self.streaming = os.environ.get("AUDIO_ENCODER_STREAMING", "false").lower() in ( "1", "true", "yes", "on" ) self.streaming_chunk_ms = float( os.environ.get("AUDIO_ENCODER_STREAMING_CHUNK_MS", "640") ) if self.streaming_chunk_ms <= 0: raise ValueError( "AUDIO_ENCODER_STREAMING_CHUNK_MS must be positive, " f"got {self.streaming_chunk_ms}" ) # Fbank frame shift is 10 ms and the two convolutions downsample time. conv_downsample = self.conv1.stride[0] * self.conv2.stride[0] self.streaming_chunk_frames = max( 1, round(self.streaming_chunk_ms / (10.0 * conv_downsample)) ) if self.streaming: logger.info( "Using streaming audio encoder attention: chunk=%s ms " "(%s encoder frames)", self.streaming_chunk_ms, self.streaming_chunk_frames, ) # debug def _streaming_attention_mask(self, lengths, sequence_length, device): """Return [B, T, T] bool mask; True entries may be attended.""" positions = torch.arange(sequence_length, device=device) chunk_ids = torch.div( positions, self.streaming_chunk_frames, rounding_mode="floor" ) block_causal = chunk_ids.unsqueeze(0) <= chunk_ids.unsqueeze(1) valid_keys = positions.unsqueeze(0) < lengths.to(device=device).unsqueeze(1) return block_causal.unsqueeze(0) & valid_keys[:, None, :] def forward( self, x: torch.Tensor, ilens: torch.Tensor = None, **kwargs, ): use_padmask = self.use_padmask x = F.gelu(self.conv1(x)) x = F.gelu(self.conv2(x)) x = x.permute(0, 2, 1) n_frames = x.size(1) max_pos = n_frames if ilens is not None: if self.downsample_rate == 4: olens = ( 1 + (ilens - self.conv1.kernel_size[0] + 2 * self.conv1.padding[0]) // self.conv1.stride[0] ) else: olens = ilens olens = ( 1 + (olens - self.conv2.kernel_size[0] + 2 * self.conv2.padding[0]) // self.conv2.stride[0] ) olens = torch.clamp(olens, max=max_pos) else: olens = None if self.streaming: if olens is None: olens = torch.full( (x.shape[0],), max_pos, dtype=torch.long, device=x.device ) padding_mask = self._streaming_attention_mask( olens, max_pos, x.device ) elif use_padmask and olens is not None: padding_mask = (~make_pad_mask(olens)[:, None, :]).to(torch.bool).to(x.device) else: padding_mask = None device = x.device seq_length = x.shape[1] position_ids = torch.arange(0, seq_length, dtype=torch.long, device=device) position_ids = position_ids.unsqueeze(0).view(-1, seq_length) for layer, block in enumerate(self.blocks): x = block(x, mask=padding_mask, position_ids=position_ids) x = self.ln_post(x) if ilens is None: return x else: return x, olens # --------------------------------------------------------------------------- # block640-lookahead160 streaming patch, verbatim from the authors' # S3-S2ST/inference/streaming/block640_lookahead160_encoder.py # --------------------------------------------------------------------------- from types import MethodType # noqa: E402 def _block_streaming_fsmn( inputs: torch.Tensor, mask: torch.Tensor | None, conv: nn.Conv1d, chunk_frames: int, right_context_frames: int, ) -> torch.Tensor: """Match training's tap-preserving FSMN with an exact per-block cutoff.""" batch, time, dim = inputs.shape kernel = int(conv.kernel_size[0]) if kernel % 2 != 1: raise ValueError(f"block FSMN requires an odd kernel, got {kernel}") half = kernel // 2 if not 0 <= right_context_frames <= half: raise ValueError( "FSMN right context exceeds the offline taps: " f"right={right_context_frames} half={half}" ) valid = None if mask is not None: valid = mask.reshape(batch, -1, 1).to(dtype=inputs.dtype) inputs = inputs * valid num_chunks = (time + chunk_frames - 1) // chunk_frames padded_time = num_chunks * chunk_frames values = inputs.transpose(1, 2) if padded_time != time: values = F.pad(values, (0, padded_time - time)) padded = F.pad(values, (half, half)) context = 2 * half + chunk_frames windows = padded.unfold(-1, context, chunk_frames) windows = windows.permute(0, 2, 1, 3).contiguous() boundary_gate = torch.ones( context, dtype=windows.dtype, device=windows.device ) boundary_gate[half + chunk_frames + right_context_frames :] = 0 windows = windows * boundary_gate.view(1, 1, 1, context) windows = windows.reshape(batch * num_chunks, dim, context) output = F.conv1d( windows, conv.weight, conv.bias, stride=1, padding=0, dilation=conv.dilation, groups=conv.groups, ) output = output.reshape(batch, num_chunks, dim, chunk_frames) output = output.permute(0, 2, 1, 3).reshape(batch, dim, padded_time) output = output[..., :time].transpose(1, 2) + inputs if valid is not None: output = output * valid return output def _block_attention_allow_mask( lengths: torch.Tensor, sequence_length: int, chunk_frames: int, lookahead_frames: int, device: torch.device, ) -> torch.Tensor: """Build the exact [B,Q,K] allow-mask used by training.""" positions = torch.arange(sequence_length, device=device) chunk_ids = torch.div(positions, chunk_frames, rounding_mode="floor") chunk_end = (chunk_ids + 1) * chunk_frames - 1 visible = positions.unsqueeze(0) <= ( chunk_end + lookahead_frames ).unsqueeze(1) valid = positions.unsqueeze(0) < lengths.to(device=device).unsqueeze(1) return visible.unsqueeze(0) & valid.unsqueeze(1) & valid.unsqueeze(2) def _streaming_qkv_attention(attn, q, k, v, mask=None, **kwargs): batch, time, dim = q.shape scale = (dim // attn.attention_heads) ** -0.5 q = q.view(batch, time, attn.attention_heads, -1).permute(0, 2, 1, 3) k = k.view(batch, time, attn.attention_heads, -1).permute(0, 2, 1, 3) v = v.view(batch, time, attn.attention_heads, -1).permute(0, 2, 1, 3) position_ids = kwargs.get("position_ids") cos, sin = attn.rotary_emb(v, seq_len=v.shape[-2]) q, k = apply_rotary_pos_emb(q, k, cos, sin, position_ids) attention_mask = kwargs.get("attention_mask") if attention_mask is None: attention_mask = mask if attention_mask is not None: attention_mask = attention_mask.unsqueeze(1).to(torch.bool) output = F.scaled_dot_product_attention( q, k, v, attn_mask=attention_mask, dropout_p=0.0, is_causal=False, scale=scale, ) if mask is not None: valid_queries = mask.to(torch.bool).unsqueeze(-1) output = output.masked_fill(valid_queries.logical_not(), 0.0) return output.transpose(1, 2).flatten(start_dim=2), None def enable_block640_lookahead160_encoder( encoder: nn.Module, chunk_ms: float = 640.0, lookahead_ms: float = 160.0, ) -> None: """Patch vLLM's encoder to match the isolated training implementation.""" if chunk_ms <= 0 or lookahead_ms < 0: raise ValueError( f"invalid block/lookahead: chunk={chunk_ms}, lookahead={lookahead_ms}" ) conv_downsample = encoder.conv1.stride[0] * encoder.conv2.stride[0] chunk_frames = max(1, round(chunk_ms / (10.0 * conv_downsample))) lookahead_frames = round(lookahead_ms / (10.0 * conv_downsample)) if chunk_frames != 16 or lookahead_frames != 4: raise ValueError( "block640-lookahead160 requires 16+4 encoder frames, got " f"{chunk_frames}+{lookahead_frames}" ) for layer, block in enumerate(encoder.blocks): attn = block.attn attn.qkv_attention = MethodType(_streaming_qkv_attention, attn) right_context = lookahead_frames if layer == 0 else 0 def block_fsmn(this, inputs, mask=None, *, _right=right_context): output = _block_streaming_fsmn( inputs, mask, this.fsmn_block, chunk_frames, _right, ) return this.dropout(output) attn.fsmn = MethodType(block_fsmn, attn) def streaming_forward(module, x, ilens=None, **_kwargs): # Preserve the offline convolution and its pretrained centered taps. x = F.gelu(module.conv1(x)) x = F.gelu(module.conv2(x)).permute(0, 2, 1) time = x.size(1) if ilens is None: olens = torch.full( (x.size(0),), time, dtype=torch.long, device=x.device ) else: olens = 1 + ( ilens - module.conv1.kernel_size[0] + 2 * module.conv1.padding[0] ) // module.conv1.stride[0] olens = 1 + ( olens - module.conv2.kernel_size[0] + 2 * module.conv2.padding[0] ) // module.conv2.stride[0] olens = torch.clamp(olens, max=time) positions = torch.arange(time, device=x.device) valid = positions.unsqueeze(0) < olens.unsqueeze(1) padding_mask = valid.unsqueeze(1) position_ids = positions.unsqueeze(0) for layer, block in enumerate(module.blocks): layer_lookahead = lookahead_frames if layer == 0 else 0 attention_mask = _block_attention_allow_mask( olens, time, chunk_frames, layer_lookahead, x.device, ) x = block( x, mask=padding_mask, position_ids=position_ids, attention_mask=attention_mask, ) x = module.ln_post(x) return x if ilens is None else (x, olens) encoder.forward = MethodType(streaming_forward, encoder) encoder.streaming_chunk_ms = chunk_ms encoder.streaming_chunk_frames = chunk_frames encoder.streaming_lookahead_ms = lookahead_ms encoder.streaming_lookahead_frames = lookahead_frames print( "[BLOCK640 LOOKAHEAD160 INFERENCE] " "offline-conv + tap-preserving-FSMN; " "chunk=640ms; layer0-lookahead=160ms; later-layer-lookahead=0", flush=True, )
# Audio encoder for S3-S2ST, extracted from the authors' vLLM fork
# (S3-S2ST/vendor/vllm/model_executor/models/omnidrca2_encoder.py, Apache-2.0,
# adapted from FunAudioLLM/Fun-Audio-Chat). vLLM-specific imports removed.
import logging
import math
import os
from typing import Optional, Tuple
import numpy as np
import torch
import torch.nn.functional as F
from torch import Tensor, nn
logger = logging.getLogger(__name__)
def make_pad_mask(lengths, xs=None, length_dim=-1, maxlen=None):
"""Make mask tensor containing indices of padded part.
Args:
lengths (LongTensor or List): Batch of lengths (B,).
xs (Tensor, optional): The reference tensor.
If set, masks will be the same shape as this tensor.
length_dim (int, optional): Dimension indicator of the above tensor.
See the example.
Returns:
Tensor: Mask tensor containing indices of padded part.
dtype=torch.uint8 in PyTorch 1.2-
dtype=torch.bool in PyTorch 1.2+ (including 1.2)
Examples:
With only lengths.
>>> lengths = [5, 3, 2]
>>> make_pad_mask(lengths)
masks = [[0, 0, 0, 0 ,0],
[0, 0, 0, 1, 1],
[0, 0, 1, 1, 1]]
With the reference tensor.
>>> xs = torch.zeros((3, 2, 4))
>>> make_pad_mask(lengths, xs)
tensor([[[0, 0, 0, 0],
[0, 0, 0, 0]],
[[0, 0, 0, 1],
[0, 0, 0, 1]],
[[0, 0, 1, 1],
[0, 0, 1, 1]]], dtype=torch.uint8)
>>> xs = torch.zeros((3, 2, 6))
>>> make_pad_mask(lengths, xs)
tensor([[[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1]],
[[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1]],
[[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1]]], dtype=torch.uint8)
With the reference tensor and dimension indicator.
>>> xs = torch.zeros((3, 6, 6))
>>> make_pad_mask(lengths, xs, 1)
tensor([[[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1]],
[[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1]],
[[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1]]], dtype=torch.uint8)
>>> make_pad_mask(lengths, xs, 2)
tensor([[[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1]],
[[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1]],
[[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1]]], dtype=torch.uint8)