Download reader.py from whiletruelearn/dulu-malayalam-ctc-778h: direct link, hf CLI and curl.
- Browser
- Download file 2.76 kB
-
https://huggingface.co/whiletruelearn/dulu-malayalam-ctc-778h/resolve/main/reader.py
- Command line
-
hf download hf://whiletruelearn/dulu-malayalam-ctc-778h/reader.py
-
curl -L -o reader.py https://huggingface.co/whiletruelearn/dulu-malayalam-ctc-778h/resolve/main/reader.py
2.76 kB
| """Cached-feature version of the six-layer reader described by qnguyen3/gemma-asr. | |
| Upstream reference: 343c86faab52fdd09c312945ad21b8f8f15e76ee. | |
| The acoustic encoder is frozen; all 13 representations remain available to the | |
| trainable layer mixture. No feature augmentation is used in this tiny pilot. | |
| """ | |
| import torch | |
| from torch import nn | |
| from torch.nn import functional as F | |
| class Block(nn.Module): | |
| def __init__(self, d, dropout): | |
| super().__init__() | |
| self.norm1 = nn.LayerNorm(d) | |
| self.qkv = nn.Linear(d, 3 * d) | |
| self.proj = nn.Linear(d, d) | |
| self.norm2 = nn.LayerNorm(d) | |
| self.ff = nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(), | |
| nn.Dropout(dropout), nn.Linear(4 * d, d)) | |
| self.drop = nn.Dropout(dropout) | |
| self.dropout = dropout | |
| def forward(self, x, mask): | |
| b, t, d = x.shape | |
| q, k, v = self.qkv(self.norm1(x)).view(b, t, 3, 8, d // 8).permute(2, 0, 3, 1, 4) | |
| a = F.scaled_dot_product_attention(q, k, v, attn_mask=mask[:, None, None, :], | |
| dropout_p=self.dropout if self.training else 0) | |
| x = x + self.drop(self.proj(a.transpose(1, 2).reshape(b, t, d))) | |
| return x + self.drop(self.ff(self.norm2(x))) | |
| class Reader(nn.Module): | |
| def __init__(self, encoder_dim, encoder_states, embeddings, cfg): | |
| super().__init__() | |
| d = cfg["d_model"] | |
| self.layer_logits = nn.Parameter(torch.zeros(encoder_states)) | |
| self.in_norm = nn.LayerNorm(encoder_dim) | |
| self.in_proj = nn.Linear(encoder_dim, d) | |
| self.pos_conv = nn.Conv1d(d, d, 15, padding=7, groups=d) | |
| self.blocks = nn.ModuleList([Block(d, cfg["dropout"]) for _ in range(cfg["reader_layers"])]) | |
| self.out_norm = nn.LayerNorm(d) | |
| self.to_text = nn.Linear(d, embeddings.shape[1]) | |
| self.unit_rows = nn.Parameter(F.normalize(embeddings.float(), dim=-1)) | |
| self.special_rows = nn.Parameter(torch.randn(2, embeddings.shape[1]) / embeddings.shape[1] ** 0.5) | |
| self.head_bias = nn.Parameter(torch.zeros(len(embeddings) + 2)) | |
| def forward(self, states, lengths): | |
| # states: batch, encoder layer, frames, channels | |
| weights = self.layer_logits.softmax(0) | |
| h = (states * weights[None, :, None, None]).sum(1) | |
| mask = torch.arange(h.shape[1], device=h.device)[None, :] < lengths[:, None] | |
| x = self.in_proj(self.in_norm(h)) | |
| x = x + F.gelu(self.pos_conv((x * mask[..., None]).transpose(1, 2)).transpose(1, 2)) | |
| for block in self.blocks: | |
| x = block(x, mask) | |
| text = self.to_text(self.out_norm(x)) | |
| rows = torch.cat([self.special_rows, self.unit_rows]) | |
| return text @ rows.T + self.head_bias | |