Spaces:
Sleeping
Sleeping
Download src/modeling/cross_attention.py from KeisukeMiyamoto/lambda-audio-v6: direct link, hf CLI and curl.
- Browser
- Download file 2.68 kB
-
https://huggingface.co/spaces/KeisukeMiyamoto/lambda-audio-v6/resolve/main/src/modeling/cross_attention.py
- Command line
-
hf download hf://spaces/KeisukeMiyamoto/lambda-audio-v6/src/modeling/cross_attention.py
-
curl -L -o cross_attention.py https://huggingface.co/spaces/KeisukeMiyamoto/lambda-audio-v6/resolve/main/src/modeling/cross_attention.py
2.68 kB
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| class GatedCrossAttention(nn.Module): | |
| def __init__(self, hidden_size: int, num_heads: int, norm_eps: float) -> None: | |
| super().__init__() | |
| self.norm = nn.RMSNorm(hidden_size, eps=norm_eps) | |
| self.attention = nn.MultiheadAttention( | |
| embed_dim=hidden_size, | |
| num_heads=num_heads, | |
| bias=False, | |
| batch_first=True, | |
| ) | |
| self.gate = nn.Parameter(torch.zeros(())) | |
| def project_key_value(self, audio_features: torch.Tensor) -> torch.Tensor: | |
| # --------------------------------------------------------- | |
| # Project audio once and retain the checkpoint-compatible MHA weights. | |
| # --------------------------------------------------------- | |
| _, key_weight, value_weight = self.attention.in_proj_weight.chunk(3) | |
| batch_size, audio_length, _ = audio_features.shape | |
| projection_shape = ( | |
| batch_size, | |
| audio_length, | |
| self.attention.num_heads, | |
| self.attention.head_dim, | |
| ) | |
| key = F.linear(audio_features, key_weight).view(projection_shape) | |
| value = F.linear(audio_features, value_weight).view(projection_shape) | |
| return torch.stack( | |
| (key.transpose(1, 2), value.transpose(1, 2)), | |
| dim=1, | |
| ) | |
| def forward( | |
| self, | |
| hidden_states: torch.Tensor, | |
| audio_key_values: torch.Tensor, | |
| audio_padding_mask: torch.BoolTensor, | |
| ) -> torch.Tensor: | |
| # --------------------------------------------------------- | |
| # Attend from text queries to the precomputed audio keys and values. | |
| # --------------------------------------------------------- | |
| query_weight, _, _ = self.attention.in_proj_weight.chunk(3) | |
| batch_size, text_length, hidden_size = hidden_states.shape | |
| query = F.linear(self.norm(hidden_states), query_weight) | |
| query = query.view( | |
| batch_size, | |
| text_length, | |
| self.attention.num_heads, | |
| self.attention.head_dim, | |
| ).transpose(1, 2) | |
| attention_output = F.scaled_dot_product_attention( | |
| query, | |
| audio_key_values[:, 0], | |
| audio_key_values[:, 1], | |
| attn_mask=~audio_padding_mask[:, None, None, :], | |
| dropout_p=0.0, | |
| ) | |
| attention_output = attention_output.transpose(1, 2).reshape( | |
| batch_size, | |
| text_length, | |
| hidden_size, | |
| ) | |
| attention_output = self.attention.out_proj(attention_output) | |
| return hidden_states + torch.tanh(self.gate) * attention_output | |