Spaces:
Running on Zero
Running on Zero
Download s3_encoder.py from hugging-apps/s3-s2st: direct link, hf CLI and curl.
- Browser
- Download file 39.1 kB
-
https://huggingface.co/spaces/hugging-apps/s3-s2st/resolve/main/s3_encoder.py
- Command line
-
hf download hf://spaces/hugging-apps/s3-s2st/s3_encoder.py
-
curl -L -o s3_encoder.py https://huggingface.co/spaces/hugging-apps/s3-s2st/resolve/main/s3_encoder.py
39.1 kB
# Audio encoder for S3-S2ST, extracted from the authors' vLLM fork
# (S3-S2ST/vendor/vllm/model_executor/models/omnidrca2_encoder.py, Apache-2.0,
# adapted from FunAudioLLM/Fun-Audio-Chat). vLLM-specific imports removed.
import logging
import math
import os
from typing import Optional, Tuple
import numpy as np
import torch
import torch.nn.functional as F
from torch import Tensor, nn
logger = logging.getLogger(__name__)
def make_pad_mask(lengths, xs=None, length_dim=-1, maxlen=None):
"""Make mask tensor containing indices of padded part.
Args:
lengths (LongTensor or List): Batch of lengths (B,).
xs (Tensor, optional): The reference tensor.
If set, masks will be the same shape as this tensor.
length_dim (int, optional): Dimension indicator of the above tensor.
See the example.
Returns:
Tensor: Mask tensor containing indices of padded part.
dtype=torch.uint8 in PyTorch 1.2-
dtype=torch.bool in PyTorch 1.2+ (including 1.2)
Examples:
With only lengths.
>>> lengths = [5, 3, 2]
>>> make_pad_mask(lengths)
masks = [[0, 0, 0, 0 ,0],
[0, 0, 0, 1, 1],
[0, 0, 1, 1, 1]]
With the reference tensor.
>>> xs = torch.zeros((3, 2, 4))
>>> make_pad_mask(lengths, xs)
tensor([[[0, 0, 0, 0],
[0, 0, 0, 0]],
[[0, 0, 0, 1],
[0, 0, 0, 1]],
[[0, 0, 1, 1],
[0, 0, 1, 1]]], dtype=torch.uint8)
>>> xs = torch.zeros((3, 2, 6))
>>> make_pad_mask(lengths, xs)
tensor([[[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1]],
[[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1]],
[[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1]]], dtype=torch.uint8)
With the reference tensor and dimension indicator.
>>> xs = torch.zeros((3, 6, 6))
>>> make_pad_mask(lengths, xs, 1)
tensor([[[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1]],
[[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1]],
[[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1]]], dtype=torch.uint8)
>>> make_pad_mask(lengths, xs, 2)
tensor([[[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1]],
[[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1]],
[[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1]]], dtype=torch.uint8)
"""
if length_dim == 0:
raise ValueError("length_dim cannot be 0: {}".format(length_dim))
if not isinstance(lengths, list):
lengths = lengths.tolist()
bs = int(len(lengths))
if maxlen is None:
if xs is None:
maxlen = int(max(lengths))
else:
maxlen = xs.size(length_dim)
else:
assert xs is None
assert maxlen >= int(max(lengths))
seq_range = torch.arange(0, maxlen, dtype=torch.int64)
seq_range_expand = seq_range.unsqueeze(0).expand(bs, maxlen)
seq_length_expand = seq_range_expand.new(lengths).unsqueeze(-1)
mask = seq_range_expand >= seq_length_expand
if xs is not None:
assert xs.size(0) == bs, (xs.size(0), bs)
if length_dim < 0:
length_dim = xs.dim() + length_dim
# ind = (:, None, ..., None, :, , None, ..., None)
ind = tuple(slice(None) if i in (0, length_dim) else None for i in range(xs.dim()))
mask = mask[ind].expand_as(xs).to(xs.device)
return mask
class LayerNorm(nn.LayerNorm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def forward(self, input):
output = F.layer_norm(
input.float(),
self.normalized_shape,
self.weight.float() if self.weight is not None else None,
self.bias.float() if self.bias is not None else None,
self.eps,
)
return output.type_as(input)
# # encoder: SenseVoiceEncoder
# encoder_conf = {
# "input_size": 128,
# "attention_heads": 20,
# "linear_units": 1280,
# "num_blocks": 32,
# "dropout_rate": 0.1,
# "positional_dropout_rate": 0.1,
# "attention_dropout_rate": 0.1,
# "kernel_size": 31,
# "sanm_shfit": 0,
# "att_type": "self_att_fsmn_sdpa",
# "downsample_rate": 4,
# "use_padmask": True,
# "max_position_embeddings": 2048,
# "rope_theta": 10000,
# "freeze": False,
# "freeze_exclude_key": "none",
# }
# Copied from transformers.models.llama.modeling_llama.rotate_half
def rotate_half(x):
"""Rotates half the hidden dims of the input."""
x1 = x[..., : x.shape[-1] // 2]
x2 = x[..., x.shape[-1] // 2 :]
return torch.cat((-x2, x1), dim=-1)
# Copied from transformers.models.mistral.modeling_mistral.apply_rotary_pos_emb
def apply_rotary_pos_emb(q, k, cos, sin, position_ids, unsqueeze_dim=1):
"""Applies Rotary Position Embedding to the query and key tensors.
Args:
q (`torch.Tensor`): The query tensor.
k (`torch.Tensor`): The key tensor.
cos (`torch.Tensor`): The cosine part of the rotary embedding.
sin (`torch.Tensor`): The sine part of the rotary embedding.
position_ids (`torch.Tensor`):
The position indices of the tokens corresponding to the query and key tensors. For example, this can be
used to pass offsetted position ids when working with a KV-cache.
unsqueeze_dim (`int`, *optional*, defaults to 1):
The 'unsqueeze_dim' argument specifies the dimension along which to unsqueeze cos[position_ids] and
sin[position_ids] so that they can be properly broadcasted to the dimensions of q and k. For example, note
that cos[position_ids] and sin[position_ids] have the shape [batch_size, seq_len, head_dim]. Then, if q and
k have the shape [batch_size, heads, seq_len, head_dim], then setting unsqueeze_dim=1 makes
cos[position_ids] and sin[position_ids] broadcastable to the shapes of q and k. Similarly, if q and k have
the shape [batch_size, seq_len, heads, head_dim], then set unsqueeze_dim=2.
Returns:
`tuple(torch.Tensor)` comprising of the query and key tensors rotated using the Rotary Position Embedding.
"""
cos = cos[position_ids].unsqueeze(unsqueeze_dim)
sin = sin[position_ids].unsqueeze(unsqueeze_dim)
q_embed = (q * cos) + (rotate_half(q) * sin)
k_embed = (k * cos) + (rotate_half(k) * sin)
return q_embed, k_embed
class RotaryEmbedding(nn.Module):
def __init__(self, dim, max_position_embeddings=2048, base=10000, device=None):
super().__init__()
self.dim = dim
self.max_position_embeddings = max_position_embeddings
self.base = base
inv_freq = 1.0 / (
self.base
** (torch.arange(0, self.dim, 2, dtype=torch.int64).float().to(device) / self.dim)
)
self.register_buffer("inv_freq", inv_freq, persistent=False)
# Build here to make `torch.jit.trace` work.
self._set_cos_sin_cache(
seq_len=max_position_embeddings,
device=self.inv_freq.device,
dtype=torch.get_default_dtype(),
)
def _set_cos_sin_cache(self, seq_len, device, dtype):
self.max_seq_len_cached = seq_len
t = torch.arange(self.max_seq_len_cached, device=device, dtype=torch.int64).type_as(
self.inv_freq
)
freqs = torch.outer(t, self.inv_freq)
# Different from paper, but it uses a different permutation in order to obtain the same calculation
emb = torch.cat((freqs, freqs), dim=-1)
self.register_buffer("cos_cached", emb.cos().to(dtype), persistent=False)
self.register_buffer("sin_cached", emb.sin().to(dtype), persistent=False)
def forward(self, x, seq_len=None):
# x: [bs, num_attention_heads, seq_len, head_size]
if seq_len > self.max_seq_len_cached:
self._set_cos_sin_cache(seq_len=seq_len, device=x.device, dtype=x.dtype)
return (
self.cos_cached[:seq_len].to(dtype=x.dtype),
self.sin_cached[:seq_len].to(dtype=x.dtype),
)
class Linear(nn.Linear):
def forward(self, x: Tensor) -> Tensor:
return F.linear(
x,
self.weight.to(x.dtype),
None if self.bias is None else self.bias.to(x.dtype),
)
class MultiHeadAttention(nn.Module):
def __init__(self, n_state: int, n_head: int):
super().__init__()
self.n_head = n_head
self.query = Linear(n_state, n_state)
self.key = Linear(n_state, n_state, bias=False)
self.value = Linear(n_state, n_state)
self.out = Linear(n_state, n_state)
def forward(
self,
x: Tensor,
xa: Optional[Tensor] = None,
mask: Optional[Tensor] = None,
kv_cache: Optional[dict] = None,
**kwargs,
):
is_pad_mask = kwargs.get("is_pad_mask", False)
q = self.query(x)
if kv_cache is None or xa is None or self.key not in kv_cache:
# hooks, if installed (i.e. kv_cache is not None), will prepend the cached kv tensors;
# otherwise, perform key/value projections for self- or cross-attention as usual.
k = self.key(x if xa is None else xa)
v = self.value(x if xa is None else xa)
else:
# for cross-attention, calculate keys and values once and reuse in subsequent calls.
k = kv_cache[self.key]
v = kv_cache[self.value]
wv, qk = self.qkv_attention(q, k, v, mask, is_pad_mask=is_pad_mask)
return self.out(wv), qk
def qkv_attention(
self,
q: Tensor,
k: Tensor,
v: Tensor,
mask: Optional[Tensor] = None,
**kwargs,
):
is_pad_mask = kwargs.get("is_pad_mask", False)
n_batch, n_ctx, n_state = q.shape
scale = (n_state // self.n_head) ** -0.25
q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) * scale
k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 3, 1) * scale
v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
qk = q @ k
if mask is not None:
if not is_pad_mask:
qk = qk + mask[:n_ctx, :n_ctx]
else:
mask = mask.unsqueeze(1).eq(0) # (batch, 1, t, 1)
min_value = -float(
"inf"
) # min_value = float(np.finfo(torch.tensor(0, dtype=qk.dtype).numpy().dtype).min)
qk = qk.masked_fill(mask, min_value)
qk = qk.float()
w = F.softmax(qk, dim=-1).to(q.dtype)
if mask is not None and is_pad_mask:
w = w.masked_fill(mask, 0.0)
return (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2), qk.detach()
class MultiHeadAttentionSdpa(nn.Module):
def __init__(self, n_state: int, n_head: int):
super().__init__()
self.n_head = n_head
self.query = Linear(n_state, n_state)
self.key = Linear(n_state, n_state, bias=False)
self.value = Linear(n_state, n_state)
self.out = Linear(n_state, n_state)
def forward(
self,
x: Tensor,
xa: Optional[Tensor] = None,
mask: Optional[Tensor] = None,
kv_cache: Optional[dict] = None,
**kwargs,
):
is_pad_mask = kwargs.get("is_pad_mask", False)
q = self.query(x)
if kv_cache is None or xa is None or self.key not in kv_cache:
# hooks, if installed (i.e. kv_cache is not None), will prepend the cached kv tensors;
# otherwise, perform key/value projections for self- or cross-attention as usual.
k = self.key(x if xa is None else xa)
v = self.value(x if xa is None else xa)
else:
# for cross-attention, calculate keys and values once and reuse in subsequent calls.
k = kv_cache[self.key]
v = kv_cache[self.value]
wv, qk = self.qkv_attention(q, k, v, mask, is_pad_mask=is_pad_mask, is_causal=False)
return self.out(wv), qk
def qkv_attention(
self,
q: Tensor,
k: Tensor,
v: Tensor,
mask: Optional[Tensor] = None,
**kwargs,
):
is_pad_mask = kwargs.get("is_pad_mask", False)
is_causal = kwargs.get("is_causal", False)
n_batch, n_ctx, n_state = q.shape
scale = (n_state // self.n_head) ** -0.5
q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
if mask is not None:
if not is_pad_mask:
mask = None
is_causal = True
else:
mask = mask.unsqueeze(1).to(torch.bool) # (batch, 1, 1, t)
attn_output = torch.nn.functional.scaled_dot_product_attention(
q,
k,
v,
attn_mask=mask,
dropout_p=0.0,
is_causal=is_causal,
scale=scale,
)
if mask is not None:
attn_output = attn_output.masked_fill(mask.transpose(2, 3).logical_not(), 0.0)
attn_output = attn_output.transpose(1, 2)
attn_output = attn_output.flatten(start_dim=2)
return attn_output, None
class MultiHeadAttentionRoPE(nn.Module):
def __init__(self, linear_units: int, attention_heads: int, **kwargs):
super().__init__()
self.attention_heads = attention_heads
self.query = Linear(linear_units, linear_units)
self.key = Linear(linear_units, linear_units, bias=False)
self.value = Linear(linear_units, linear_units)
self.out = Linear(linear_units, linear_units)
self.rotary_emb = RotaryEmbedding(
linear_units // attention_heads,
max_position_embeddings=kwargs.get("max_position_embeddings", 2048),
base=kwargs.get("rope_theta", 10000),
)
def forward(
self,
x: Tensor,
mask: Optional[Tensor] = None,
**kwargs,
):
q = self.query(x)
k = self.key(x)
v = self.value(x)
wv, qk = self.qkv_attention(q, k, v, mask, **kwargs)
return self.out(wv), qk
def qkv_attention(
self,
q: Tensor,
k: Tensor,
v: Tensor,
mask: Optional[Tensor] = None,
**kwargs,
):
n_batch, n_ctx, n_state = q.shape
scale = (n_state // self.n_head) ** -0.25
q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3) * scale
k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 3, 1) * scale
v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
position_ids = kwargs.get("position_ids", None)
kv_seq_len = v.shape[-2]
cos, sin = self.rotary_emb(v, seq_len=kv_seq_len)
q, k = apply_rotary_pos_emb(q, k, cos, sin, position_ids)
qk = q @ k
if mask is not None:
mask = mask.unsqueeze(1).eq(0) # (batch, 1, t, 1)
min_value = -float(
"inf"
) # min_value = float(np.finfo(torch.tensor(0, dtype=qk.dtype).numpy().dtype).min)
qk = qk.masked_fill(mask, min_value)
qk = qk.float()
w = F.softmax(qk, dim=-1).to(q.dtype)
if mask is not None:
w = w.masked_fill(mask, 0.0)
return (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2), qk.detach()
class MultiHeadAttentionSdpaRoPE(nn.Module):
def __init__(self, linear_units: int, attention_heads: int, **kwargs):
super().__init__()
self.attention_heads = attention_heads
self.query = Linear(linear_units, linear_units)
self.key = Linear(linear_units, linear_units, bias=False)
self.value = Linear(linear_units, linear_units)
self.out = Linear(linear_units, linear_units)
self.rotary_emb = RotaryEmbedding(
linear_units // attention_heads,
max_position_embeddings=kwargs.get("max_position_embeddings", 2048),
base=kwargs.get("rope_theta", 10000),
)
def forward(
self,
x: Tensor,
mask: Optional[Tensor] = None,
**kwargs,
):
q = self.query(x)
k = self.key(x)
v = self.value(x)
wv, qk = self.qkv_attention(q, k, v, mask, **kwargs)
return self.out(wv), qk
def qkv_attention(
self,
q: Tensor,
k: Tensor,
v: Tensor,
mask: Optional[Tensor] = None,
**kwargs,
):
is_causal = kwargs.get("is_causal", False)
n_batch, n_ctx, n_state = q.shape
scale = (n_state // self.n_head) ** -0.5
q = q.view(*q.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
k = k.view(*k.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
v = v.view(*v.shape[:2], self.n_head, -1).permute(0, 2, 1, 3)
position_ids = kwargs.get("position_ids", None)
kv_seq_len = v.shape[-2]
cos, sin = self.rotary_emb(v, seq_len=kv_seq_len)
q, k = apply_rotary_pos_emb(q, k, cos, sin, position_ids)
if mask is not None:
mask = mask.unsqueeze(1).to(torch.bool) # (batch, 1, 1, t)
attn_output = torch.nn.functional.scaled_dot_product_attention(
q,
k,
v,
attn_mask=mask,
dropout_p=0.0,
is_causal=is_causal,
scale=scale,
)
if mask is not None:
attn_output = attn_output.masked_fill(mask.transpose(2, 3).logical_not(), 0.0)
attn_output = attn_output.transpose(1, 2)
attn_output = attn_output.flatten(start_dim=2)
return attn_output, None
class MultiHeadAttentionFSMNRoPE(nn.Module):
def __init__(self, linear_units: int, attention_heads: int, **kwargs):
super().__init__()
self.attention_heads = attention_heads
self.query = Linear(linear_units, linear_units)
self.key = Linear(linear_units, linear_units, bias=False)
self.value = Linear(linear_units, linear_units)
self.out = Linear(linear_units, linear_units)
self.rotary_emb = RotaryEmbedding(
linear_units // attention_heads,
max_position_embeddings=kwargs.get("max_position_embeddings", 2048),
base=kwargs.get("rope_theta", 10000),
)
self.fsmn_block = nn.Conv1d(
linear_units,
linear_units,
kwargs.get("kernel_size", 15),
stride=1,
padding=0,
groups=linear_units,
bias=False,
)
# padding
left_padding = (kwargs.get("kernel_size", 15) - 1) // 2
left_padding = left_padding + kwargs.get("sanm_shfit", 0)
right_padding = kwargs.get("kernel_size", 15) - 1 - left_padding
self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0)
self.dropout = torch.nn.Dropout(kwargs.get("dropout_rate", 0.0))
def fsmn(self, inputs, mask):
b, t, d = inputs.size()
if mask is not None:
mask = torch.reshape(mask, (b, -1, 1))
inputs = inputs * mask
x = inputs.transpose(1, 2)
x = self.pad_fn(x)
x = self.fsmn_block(x)
x = x.transpose(1, 2) + inputs
x = self.dropout(x)
if mask is not None:
x = x * mask
return x
def forward(
self,
x: Tensor,
mask: Optional[Tensor] = None,
**kwargs,
):
q = self.query(x)
k = self.key(x)
v = self.value(x)
memory = self.fsmn(v, mask=mask)
wv, qk = self.qkv_attention(q, k, v, mask, **kwargs)
return self.out(wv) + memory, qk
def qkv_attention(
self,
q: Tensor,
k: Tensor,
v: Tensor,
mask: Optional[Tensor] = None,
**kwargs,
):
b, t, d = q.shape
scale = (d // self.attention_heads) ** -0.5
q = q.view(*q.shape[:2], self.attention_heads, -1).permute(0, 2, 1, 3)
k = k.view(*k.shape[:2], self.attention_heads, -1).permute(0, 2, 1, 3)
v = v.view(*v.shape[:2], self.attention_heads, -1).permute(0, 2, 1, 3)
position_ids = kwargs.get("position_ids", None)
kv_seq_len = v.shape[-2]
cos, sin = self.rotary_emb(v, seq_len=kv_seq_len)
q, k = apply_rotary_pos_emb(q, k, cos, sin, position_ids)
qk = q @ k
if mask is not None:
mask = mask.unsqueeze(1).eq(0) # (batch, 1, t, 1)
min_value = -float(
"inf"
) # min_value = float(np.finfo(torch.tensor(0, dtype=qk.dtype).numpy().dtype).min)
qk = qk.masked_fill(mask, min_value)
qk = qk.float()
w = F.softmax(qk, dim=-1).to(q.dtype)
if mask is not None:
w = w.masked_fill(mask, 0.0)
return (w @ v).permute(0, 2, 1, 3).flatten(start_dim=2), qk.detach()
class MultiHeadAttentionFSMNSdpaRoPE(nn.Module):
def __init__(self, linear_units: int, attention_heads: int, **kwargs):
super().__init__()
self.attention_heads = attention_heads
self.query = Linear(linear_units, linear_units)
self.key = Linear(linear_units, linear_units, bias=False)
self.value = Linear(linear_units, linear_units)
self.out = Linear(linear_units, linear_units)
self.rotary_emb = RotaryEmbedding(
linear_units // attention_heads,
max_position_embeddings=kwargs.get("max_position_embeddings", 2048),
base=kwargs.get("rope_theta", 10000),
)
self.fsmn_block = nn.Conv1d(
linear_units,
linear_units,
kwargs.get("kernel_size", 15),
stride=1,
padding=0,
groups=linear_units,
bias=False,
)
# padding
left_padding = (kwargs.get("kernel_size", 15) - 1) // 2
left_padding = left_padding + kwargs.get("sanm_shfit", 0)
right_padding = kwargs.get("kernel_size", 15) - 1 - left_padding
self.pad_fn = nn.ConstantPad1d((left_padding, right_padding), 0.0)
self.dropout = torch.nn.Dropout(kwargs.get("dropout_rate", 0.0))
def fsmn(self, inputs, mask):
b, t, d = inputs.size() # b, t, d
if mask is not None:
# A streaming attention mask is [B, T_query, T_key]. FSMN only
# needs the per-frame validity mask, which is its diagonal.
frame_mask = (
torch.diagonal(mask, dim1=-2, dim2=-1)
if mask.dim() == 3
else torch.reshape(mask, (b, -1))
)
frame_mask = frame_mask.unsqueeze(-1) # b, t, 1
inputs = inputs * frame_mask
x = inputs.transpose(1, 2)
x = self.pad_fn(x)
x = self.fsmn_block(x)
x = x.transpose(1, 2) + inputs
x = self.dropout(x)
if mask is not None:
x = x * frame_mask
return x
def forward(
self,
x: Tensor,
mask: Optional[Tensor] = None,
**kwargs,
):
q = self.query(x)
k = self.key(x)
v = self.value(x)
memory = self.fsmn(v, mask=mask)
wv, qk = self.qkv_attention(q, k, v, mask, **kwargs)
return self.out(wv) + memory, qk
def qkv_attention(
self,
q: Tensor,
k: Tensor,
v: Tensor,
mask: Optional[Tensor] = None,
**kwargs,
):
is_causal = kwargs.get("is_causal", False)
b, t, d = q.shape
scale = (d // self.attention_heads) ** -0.5
q = q.view(*q.shape[:2], self.attention_heads, -1).permute(0, 2, 1, 3)
k = k.view(*k.shape[:2], self.attention_heads, -1).permute(0, 2, 1, 3)
v = v.view(*v.shape[:2], self.attention_heads, -1).permute(0, 2, 1, 3)
position_ids = kwargs.get("position_ids", None)
kv_seq_len = v.shape[-2]
cos, sin = self.rotary_emb(v, seq_len=kv_seq_len)
q, k = apply_rotary_pos_emb(q, k, cos, sin, position_ids)
query_mask = None
if mask is not None:
if mask.dim() == 3:
query_mask = torch.diagonal(mask, dim1=-2, dim2=-1)
else:
query_mask = mask.reshape(b, -1)
mask = mask.unsqueeze(1).to(torch.bool) # (batch, 1, *, t)
attn_output = torch.nn.functional.scaled_dot_product_attention(
q,
k,
v,
attn_mask=mask,
dropout_p=0.0,
is_causal=is_causal,
scale=scale,
)
if query_mask is not None:
attn_output = attn_output.masked_fill(
query_mask[:, None, :, None].logical_not(), 0.0
)
attn_output = attn_output.transpose(1, 2)
attn_output = attn_output.flatten(start_dim=2)
return attn_output, None
att_type_dict = {
"default": MultiHeadAttention,
"sdpa": MultiHeadAttentionSdpa,
"self_att": MultiHeadAttentionRoPE,
"self_att_sdpa": MultiHeadAttentionSdpaRoPE,
"self_att_fsmn": MultiHeadAttentionFSMNRoPE,
"self_att_fsmn_sdpa": MultiHeadAttentionFSMNSdpaRoPE,
}
class EncoderLayerSANMLarge(nn.Module):
def __init__(self, linear_units: int, attention_heads: int, **kwargs):
super().__init__()
att_type = kwargs.get("att_type", "self_att_fsmn_sdpa")
self.attn = att_type_dict[att_type](linear_units, attention_heads, **kwargs)
self.attn_ln = LayerNorm(linear_units)
n_mlp = linear_units * 4
self.mlp = nn.Sequential(
Linear(linear_units, n_mlp), nn.GELU(), Linear(n_mlp, linear_units)
)
self.mlp_ln = LayerNorm(linear_units)
def forward(
self,
x: Tensor,
mask: Optional[Tensor] = None,
**kwargs,
):
x = x + self.attn(self.attn_ln(x), mask=mask, **kwargs)[0]
x = x + self.mlp(self.mlp_ln(x))
return x
class Conv1d(nn.Conv1d):
def _conv_forward(self, x, weight, bias):
return super()._conv_forward(
x, weight.to(x.dtype), None if bias is None else bias.to(x.dtype)
)
class SenseVoiceEncoder(nn.Module):
def __init__(
self,
input_size,
linear_units: int,
attention_heads: int,
num_blocks: int,
**kwargs,
):
super().__init__()
self.conv1 = Conv1d(input_size, linear_units, kernel_size=3, stride=2, padding=1)
self.conv2 = Conv1d(linear_units, linear_units, kernel_size=3, stride=2, padding=1)
self.blocks = nn.ModuleList(
[
EncoderLayerSANMLarge(linear_units, attention_heads, **kwargs)
for _ in range(num_blocks)
]
)
self.ln_post = LayerNorm(linear_units)
self.use_padmask = kwargs.get("use_padmask", True)
self.downsample_rate = kwargs.get("downsample_rate", 4)
self.streaming = os.environ.get("AUDIO_ENCODER_STREAMING", "false").lower() in (
"1", "true", "yes", "on"
)
self.streaming_chunk_ms = float(
os.environ.get("AUDIO_ENCODER_STREAMING_CHUNK_MS", "640")
)
if self.streaming_chunk_ms <= 0:
raise ValueError(
"AUDIO_ENCODER_STREAMING_CHUNK_MS must be positive, "
f"got {self.streaming_chunk_ms}"
)
# Fbank frame shift is 10 ms and the two convolutions downsample time.
conv_downsample = self.conv1.stride[0] * self.conv2.stride[0]
self.streaming_chunk_frames = max(
1, round(self.streaming_chunk_ms / (10.0 * conv_downsample))
)
if self.streaming:
logger.info(
"Using streaming audio encoder attention: chunk=%s ms "
"(%s encoder frames)",
self.streaming_chunk_ms,
self.streaming_chunk_frames,
)
# debug
def _streaming_attention_mask(self, lengths, sequence_length, device):
"""Return [B, T, T] bool mask; True entries may be attended."""
positions = torch.arange(sequence_length, device=device)
chunk_ids = torch.div(
positions, self.streaming_chunk_frames, rounding_mode="floor"
)
block_causal = chunk_ids.unsqueeze(0) <= chunk_ids.unsqueeze(1)
valid_keys = positions.unsqueeze(0) < lengths.to(device=device).unsqueeze(1)
return block_causal.unsqueeze(0) & valid_keys[:, None, :]
def forward(
self,
x: torch.Tensor,
ilens: torch.Tensor = None,
**kwargs,
):
use_padmask = self.use_padmask
x = F.gelu(self.conv1(x))
x = F.gelu(self.conv2(x))
x = x.permute(0, 2, 1)
n_frames = x.size(1)
max_pos = n_frames
if ilens is not None:
if self.downsample_rate == 4:
olens = (
1
+ (ilens - self.conv1.kernel_size[0] + 2 * self.conv1.padding[0])
// self.conv1.stride[0]
)
else:
olens = ilens
olens = (
1
+ (olens - self.conv2.kernel_size[0] + 2 * self.conv2.padding[0])
// self.conv2.stride[0]
)
olens = torch.clamp(olens, max=max_pos)
else:
olens = None
if self.streaming:
if olens is None:
olens = torch.full(
(x.shape[0],), max_pos, dtype=torch.long, device=x.device
)
padding_mask = self._streaming_attention_mask(
olens, max_pos, x.device
)
elif use_padmask and olens is not None:
padding_mask = (~make_pad_mask(olens)[:, None, :]).to(torch.bool).to(x.device)
else:
padding_mask = None
device = x.device
seq_length = x.shape[1]
position_ids = torch.arange(0, seq_length, dtype=torch.long, device=device)
position_ids = position_ids.unsqueeze(0).view(-1, seq_length)
for layer, block in enumerate(self.blocks):
x = block(x, mask=padding_mask, position_ids=position_ids)
x = self.ln_post(x)
if ilens is None:
return x
else:
return x, olens
# ---------------------------------------------------------------------------
# block640-lookahead160 streaming patch, verbatim from the authors'
# S3-S2ST/inference/streaming/block640_lookahead160_encoder.py
# ---------------------------------------------------------------------------
from types import MethodType # noqa: E402
def _block_streaming_fsmn(
inputs: torch.Tensor,
mask: torch.Tensor | None,
conv: nn.Conv1d,
chunk_frames: int,
right_context_frames: int,
) -> torch.Tensor:
"""Match training's tap-preserving FSMN with an exact per-block cutoff."""
batch, time, dim = inputs.shape
kernel = int(conv.kernel_size[0])
if kernel % 2 != 1:
raise ValueError(f"block FSMN requires an odd kernel, got {kernel}")
half = kernel // 2
if not 0 <= right_context_frames <= half:
raise ValueError(
"FSMN right context exceeds the offline taps: "
f"right={right_context_frames} half={half}"
)
valid = None
if mask is not None:
valid = mask.reshape(batch, -1, 1).to(dtype=inputs.dtype)
inputs = inputs * valid
num_chunks = (time + chunk_frames - 1) // chunk_frames
padded_time = num_chunks * chunk_frames
values = inputs.transpose(1, 2)
if padded_time != time:
values = F.pad(values, (0, padded_time - time))
padded = F.pad(values, (half, half))
context = 2 * half + chunk_frames
windows = padded.unfold(-1, context, chunk_frames)
windows = windows.permute(0, 2, 1, 3).contiguous()
boundary_gate = torch.ones(
context, dtype=windows.dtype, device=windows.device
)
boundary_gate[half + chunk_frames + right_context_frames :] = 0
windows = windows * boundary_gate.view(1, 1, 1, context)
windows = windows.reshape(batch * num_chunks, dim, context)
output = F.conv1d(
windows,
conv.weight,
conv.bias,
stride=1,
padding=0,
dilation=conv.dilation,
groups=conv.groups,
)
output = output.reshape(batch, num_chunks, dim, chunk_frames)
output = output.permute(0, 2, 1, 3).reshape(batch, dim, padded_time)
output = output[..., :time].transpose(1, 2) + inputs
if valid is not None:
output = output * valid
return output
def _block_attention_allow_mask(
lengths: torch.Tensor,
sequence_length: int,
chunk_frames: int,
lookahead_frames: int,
device: torch.device,
) -> torch.Tensor:
"""Build the exact [B,Q,K] allow-mask used by training."""
positions = torch.arange(sequence_length, device=device)
chunk_ids = torch.div(positions, chunk_frames, rounding_mode="floor")
chunk_end = (chunk_ids + 1) * chunk_frames - 1
visible = positions.unsqueeze(0) <= (
chunk_end + lookahead_frames
).unsqueeze(1)
valid = positions.unsqueeze(0) < lengths.to(device=device).unsqueeze(1)
return visible.unsqueeze(0) & valid.unsqueeze(1) & valid.unsqueeze(2)
def _streaming_qkv_attention(attn, q, k, v, mask=None, **kwargs):
batch, time, dim = q.shape
scale = (dim // attn.attention_heads) ** -0.5
q = q.view(batch, time, attn.attention_heads, -1).permute(0, 2, 1, 3)
k = k.view(batch, time, attn.attention_heads, -1).permute(0, 2, 1, 3)
v = v.view(batch, time, attn.attention_heads, -1).permute(0, 2, 1, 3)
position_ids = kwargs.get("position_ids")
cos, sin = attn.rotary_emb(v, seq_len=v.shape[-2])
q, k = apply_rotary_pos_emb(q, k, cos, sin, position_ids)
attention_mask = kwargs.get("attention_mask")
if attention_mask is None:
attention_mask = mask
if attention_mask is not None:
attention_mask = attention_mask.unsqueeze(1).to(torch.bool)
output = F.scaled_dot_product_attention(
q,
k,
v,
attn_mask=attention_mask,
dropout_p=0.0,
is_causal=False,
scale=scale,
)
if mask is not None:
valid_queries = mask.to(torch.bool).unsqueeze(-1)
output = output.masked_fill(valid_queries.logical_not(), 0.0)
return output.transpose(1, 2).flatten(start_dim=2), None
def enable_block640_lookahead160_encoder(
encoder: nn.Module,
chunk_ms: float = 640.0,
lookahead_ms: float = 160.0,
) -> None:
"""Patch vLLM's encoder to match the isolated training implementation."""
if chunk_ms <= 0 or lookahead_ms < 0:
raise ValueError(
f"invalid block/lookahead: chunk={chunk_ms}, lookahead={lookahead_ms}"
)
conv_downsample = encoder.conv1.stride[0] * encoder.conv2.stride[0]
chunk_frames = max(1, round(chunk_ms / (10.0 * conv_downsample)))
lookahead_frames = round(lookahead_ms / (10.0 * conv_downsample))
if chunk_frames != 16 or lookahead_frames != 4:
raise ValueError(
"block640-lookahead160 requires 16+4 encoder frames, got "
f"{chunk_frames}+{lookahead_frames}"
)
for layer, block in enumerate(encoder.blocks):
attn = block.attn
attn.qkv_attention = MethodType(_streaming_qkv_attention, attn)
right_context = lookahead_frames if layer == 0 else 0
def block_fsmn(this, inputs, mask=None, *, _right=right_context):
output = _block_streaming_fsmn(
inputs,
mask,
this.fsmn_block,
chunk_frames,
_right,
)
return this.dropout(output)
attn.fsmn = MethodType(block_fsmn, attn)
def streaming_forward(module, x, ilens=None, **_kwargs):
# Preserve the offline convolution and its pretrained centered taps.
x = F.gelu(module.conv1(x))
x = F.gelu(module.conv2(x)).permute(0, 2, 1)
time = x.size(1)
if ilens is None:
olens = torch.full(
(x.size(0),), time, dtype=torch.long, device=x.device
)
else:
olens = 1 + (
ilens - module.conv1.kernel_size[0] + 2 * module.conv1.padding[0]
) // module.conv1.stride[0]
olens = 1 + (
olens - module.conv2.kernel_size[0] + 2 * module.conv2.padding[0]
) // module.conv2.stride[0]
olens = torch.clamp(olens, max=time)
positions = torch.arange(time, device=x.device)
valid = positions.unsqueeze(0) < olens.unsqueeze(1)
padding_mask = valid.unsqueeze(1)
position_ids = positions.unsqueeze(0)
for layer, block in enumerate(module.blocks):
layer_lookahead = lookahead_frames if layer == 0 else 0
attention_mask = _block_attention_allow_mask(
olens,
time,
chunk_frames,
layer_lookahead,
x.device,
)
x = block(
x,
mask=padding_mask,
position_ids=position_ids,
attention_mask=attention_mask,
)
x = module.ln_post(x)
return x if ilens is None else (x, olens)
encoder.forward = MethodType(streaming_forward, encoder)
encoder.streaming_chunk_ms = chunk_ms
encoder.streaming_chunk_frames = chunk_frames
encoder.streaming_lookahead_ms = lookahead_ms
encoder.streaming_lookahead_frames = lookahead_frames
print(
"[BLOCK640 LOOKAHEAD160 INFERENCE] "
"offline-conv + tap-preserving-FSMN; "
"chunk=640ms; layer0-lookahead=160ms; later-layer-lookahead=0",
flush=True,
)
# Audio encoder for S3-S2ST, extracted from the authors' vLLM fork
# (S3-S2ST/vendor/vllm/model_executor/models/omnidrca2_encoder.py, Apache-2.0,
# adapted from FunAudioLLM/Fun-Audio-Chat). vLLM-specific imports removed.
import logging
import math
import os
from typing import Optional, Tuple
import numpy as np
import torch
import torch.nn.functional as F
from torch import Tensor, nn
logger = logging.getLogger(__name__)
def make_pad_mask(lengths, xs=None, length_dim=-1, maxlen=None):
"""Make mask tensor containing indices of padded part.
Args:
lengths (LongTensor or List): Batch of lengths (B,).
xs (Tensor, optional): The reference tensor.
If set, masks will be the same shape as this tensor.
length_dim (int, optional): Dimension indicator of the above tensor.
See the example.
Returns:
Tensor: Mask tensor containing indices of padded part.
dtype=torch.uint8 in PyTorch 1.2-
dtype=torch.bool in PyTorch 1.2+ (including 1.2)
Examples:
With only lengths.
>>> lengths = [5, 3, 2]
>>> make_pad_mask(lengths)
masks = [[0, 0, 0, 0 ,0],
[0, 0, 0, 1, 1],
[0, 0, 1, 1, 1]]
With the reference tensor.
>>> xs = torch.zeros((3, 2, 4))
>>> make_pad_mask(lengths, xs)
tensor([[[0, 0, 0, 0],
[0, 0, 0, 0]],
[[0, 0, 0, 1],
[0, 0, 0, 1]],
[[0, 0, 1, 1],
[0, 0, 1, 1]]], dtype=torch.uint8)
>>> xs = torch.zeros((3, 2, 6))
>>> make_pad_mask(lengths, xs)
tensor([[[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1]],
[[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1]],
[[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1]]], dtype=torch.uint8)
With the reference tensor and dimension indicator.
>>> xs = torch.zeros((3, 6, 6))
>>> make_pad_mask(lengths, xs, 1)
tensor([[[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1]],
[[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1]],
[[0, 0, 0, 0, 0, 0],
[0, 0, 0, 0, 0, 0],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1],
[1, 1, 1, 1, 1, 1]]], dtype=torch.uint8)
>>> make_pad_mask(lengths, xs, 2)
tensor([[[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1],
[0, 0, 0, 0, 0, 1]],
[[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1],
[0, 0, 0, 1, 1, 1]],
[[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1],
[0, 0, 1, 1, 1, 1]]], dtype=torch.uint8)