whiletruelearn's picture
Archive selected epoch 9 combined-data CTC checkpoint and evaluation
750e4eb verified
Raw History Blame Contribute Delete
2.76 kB
"""Cached-feature version of the six-layer reader described by qnguyen3/gemma-asr.
Upstream reference: 343c86faab52fdd09c312945ad21b8f8f15e76ee.
The acoustic encoder is frozen; all 13 representations remain available to the
trainable layer mixture. No feature augmentation is used in this tiny pilot.
"""
import torch
from torch import nn
from torch.nn import functional as F
class Block(nn.Module):
def __init__(self, d, dropout):
super().__init__()
self.norm1 = nn.LayerNorm(d)
self.qkv = nn.Linear(d, 3 * d)
self.proj = nn.Linear(d, d)
self.norm2 = nn.LayerNorm(d)
self.ff = nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(),
nn.Dropout(dropout), nn.Linear(4 * d, d))
self.drop = nn.Dropout(dropout)
self.dropout = dropout
def forward(self, x, mask):
b, t, d = x.shape
q, k, v = self.qkv(self.norm1(x)).view(b, t, 3, 8, d // 8).permute(2, 0, 3, 1, 4)
a = F.scaled_dot_product_attention(q, k, v, attn_mask=mask[:, None, None, :],
dropout_p=self.dropout if self.training else 0)
x = x + self.drop(self.proj(a.transpose(1, 2).reshape(b, t, d)))
return x + self.drop(self.ff(self.norm2(x)))
class Reader(nn.Module):
def __init__(self, encoder_dim, encoder_states, embeddings, cfg):
super().__init__()
d = cfg["d_model"]
self.layer_logits = nn.Parameter(torch.zeros(encoder_states))
self.in_norm = nn.LayerNorm(encoder_dim)
self.in_proj = nn.Linear(encoder_dim, d)
self.pos_conv = nn.Conv1d(d, d, 15, padding=7, groups=d)
self.blocks = nn.ModuleList([Block(d, cfg["dropout"]) for _ in range(cfg["reader_layers"])])
self.out_norm = nn.LayerNorm(d)
self.to_text = nn.Linear(d, embeddings.shape[1])
self.unit_rows = nn.Parameter(F.normalize(embeddings.float(), dim=-1))
self.special_rows = nn.Parameter(torch.randn(2, embeddings.shape[1]) / embeddings.shape[1] ** 0.5)
self.head_bias = nn.Parameter(torch.zeros(len(embeddings) + 2))
def forward(self, states, lengths):
# states: batch, encoder layer, frames, channels
weights = self.layer_logits.softmax(0)
h = (states * weights[None, :, None, None]).sum(1)
mask = torch.arange(h.shape[1], device=h.device)[None, :] < lengths[:, None]
x = self.in_proj(self.in_norm(h))
x = x + F.gelu(self.pos_conv((x * mask[..., None]).transpose(1, 2)).transpose(1, 2))
for block in self.blocks:
x = block(x, mask)
text = self.to_text(self.out_norm(x))
rows = torch.cat([self.special_rows, self.unit_rows])
return text @ rows.T + self.head_bias