"""Cached-feature version of the six-layer reader described by qnguyen3/gemma-asr. Upstream reference: 343c86faab52fdd09c312945ad21b8f8f15e76ee. The acoustic encoder is frozen; all 13 representations remain available to the trainable layer mixture. No feature augmentation is used in this tiny pilot. """ import torch from torch import nn from torch.nn import functional as F class Block(nn.Module): def __init__(self, d, dropout): super().__init__() self.norm1 = nn.LayerNorm(d) self.qkv = nn.Linear(d, 3 * d) self.proj = nn.Linear(d, d) self.norm2 = nn.LayerNorm(d) self.ff = nn.Sequential(nn.Linear(d, 4 * d), nn.GELU(), nn.Dropout(dropout), nn.Linear(4 * d, d)) self.drop = nn.Dropout(dropout) self.dropout = dropout def forward(self, x, mask): b, t, d = x.shape q, k, v = self.qkv(self.norm1(x)).view(b, t, 3, 8, d // 8).permute(2, 0, 3, 1, 4) a = F.scaled_dot_product_attention(q, k, v, attn_mask=mask[:, None, None, :], dropout_p=self.dropout if self.training else 0) x = x + self.drop(self.proj(a.transpose(1, 2).reshape(b, t, d))) return x + self.drop(self.ff(self.norm2(x))) class Reader(nn.Module): def __init__(self, encoder_dim, encoder_states, embeddings, cfg): super().__init__() d = cfg["d_model"] self.layer_logits = nn.Parameter(torch.zeros(encoder_states)) self.in_norm = nn.LayerNorm(encoder_dim) self.in_proj = nn.Linear(encoder_dim, d) self.pos_conv = nn.Conv1d(d, d, 15, padding=7, groups=d) self.blocks = nn.ModuleList([Block(d, cfg["dropout"]) for _ in range(cfg["reader_layers"])]) self.out_norm = nn.LayerNorm(d) self.to_text = nn.Linear(d, embeddings.shape[1]) self.unit_rows = nn.Parameter(F.normalize(embeddings.float(), dim=-1)) self.special_rows = nn.Parameter(torch.randn(2, embeddings.shape[1]) / embeddings.shape[1] ** 0.5) self.head_bias = nn.Parameter(torch.zeros(len(embeddings) + 2)) def forward(self, states, lengths): # states: batch, encoder layer, frames, channels weights = self.layer_logits.softmax(0) h = (states * weights[None, :, None, None]).sum(1) mask = torch.arange(h.shape[1], device=h.device)[None, :] < lengths[:, None] x = self.in_proj(self.in_norm(h)) x = x + F.gelu(self.pos_conv((x * mask[..., None]).transpose(1, 2)).transpose(1, 2)) for block in self.blocks: x = block(x, mask) text = self.to_text(self.out_norm(x)) rows = torch.cat([self.special_rows, self.unit_rows]) return text @ rows.T + self.head_bias