Spaces:
Running
Running
File size: 6,402 Bytes
d7013e6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 | """
Grassmann Seq2Seq Model — Fully Attention-Free Encoder-Decoder.
Audio → Grassmann Encoder → Grassmann Decoder → Text
Every sequence interaction in this model happens through Plücker
coordinates on the Grassmann manifold Gr(2, r). There are zero
attention matrices anywhere in the architecture:
Encoder self-mixing: Plücker(z_t, z_{t+Δ}) — local causal windows
Decoder self-mixing: Plücker(z_t, z_{t+Δ}) — causal autoregressive
Decoder cross-mixing: Plücker(z_dec, z_enc) — monotonic alignment windows
Trained with teacher forcing and cross-entropy loss.
"""
import torch
import torch.nn as nn
from .config import GrassmannConfig
from .grassmann_model import ConvFrontend, PositionalEncoding
from .grassmann_layer import CausalGrassmannLayer
from .grassmann_decoder import GrassmannDecoder
class GrassmannEncoder(nn.Module):
"""Audio encoder: mel spectrogram → Grassmann layers → encoder states."""
def __init__(self, config: GrassmannConfig):
super().__init__()
self.frontend = ConvFrontend(config.n_mels, config.d_model)
self.pos_enc = PositionalEncoding(
config.d_model, config.max_seq_len, config.dropout
)
self.layers = nn.ModuleList([
CausalGrassmannLayer(config, i) for i in range(config.n_layers)
])
def forward(self, mel: torch.Tensor) -> torch.Tensor:
"""
Args:
mel: (batch, n_mels, time) - mel spectrogram
Returns:
(batch, T_enc, d_model) - encoder hidden states
"""
x = self.frontend(mel)
x = self.pos_enc(x)
for layer in self.layers:
x = layer(x)
return x
class GrassmannSeq2SeqModel(nn.Module):
"""
Fully attention-free encoder-decoder for audio transcription.
Architecture:
Encoder: ConvFrontend → PosEnc → N × CausalGrassmannLayer
Decoder: TokenEmbed → PosEmbed → M × (CausalGrassmann + CrossGrassmann + FFN)
Output: Linear → vocab logits
All interactions use Plücker coordinates. Zero attention matrices.
"""
# Special token IDs
SOS_ID = 0 # Start of sequence (overloads blank/pad)
EOS_ID = 0 # End of sequence (same as SOS for simplicity)
PAD_ID = 0 # Padding
def __init__(self, config: GrassmannConfig):
super().__init__()
self.config = config
self.encoder = GrassmannEncoder(config)
self.decoder = GrassmannDecoder(config)
def forward(
self,
mel: torch.Tensor,
target_tokens: torch.Tensor,
) -> torch.Tensor:
"""
Teacher-forced forward pass.
Args:
mel: (batch, n_mels, time) - input mel spectrogram
target_tokens: (batch, T_target) - target token indices
Returns:
logits: (batch, T_target, vocab_size)
"""
# Encode audio
encoder_out = self.encoder(mel) # (B, T_enc, d)
# Decode with teacher forcing
# Input to decoder is target shifted right (prepend SOS)
dec_input = self._shift_right(target_tokens)
logits = self.decoder(dec_input, encoder_out)
return logits
def _shift_right(self, tokens: torch.Tensor) -> torch.Tensor:
"""Shift tokens right and prepend SOS token for teacher forcing."""
sos = torch.full(
(tokens.shape[0], 1), self.SOS_ID,
dtype=tokens.dtype, device=tokens.device
)
# Prepend SOS, drop last token
return torch.cat([sos, tokens[:, :-1]], dim=1)
def get_encoder_output_lengths(self, input_lengths: torch.Tensor) -> torch.Tensor:
"""Calculate encoder output lengths after conv downsampling."""
lengths = input_lengths
for _ in range(2): # 2 conv layers with stride 2
lengths = (lengths + 1) // 2
return lengths
@torch.no_grad()
def greedy_decode(
self,
mel: torch.Tensor,
max_len: int = 200,
eos_id: int = 0,
repetition_penalty: float = 1.5,
no_repeat_ngram: int = 3,
) -> torch.Tensor:
"""
Autoregressive greedy decoding with repetition suppression.
Args:
mel: (1, n_mels, time) - single mel spectrogram
max_len: maximum output length
eos_id: end-of-sequence token ID
repetition_penalty: penalize tokens that already appeared (>1.0 to suppress)
no_repeat_ngram: block any n-gram that already occurred
Returns:
(seq_len,) - predicted token indices
"""
self.eval()
device = mel.device
# Encode once
encoder_out = self.encoder(mel) # (1, T_enc, d)
# Start with SOS token
generated = [self.SOS_ID]
for _ in range(max_len):
# Build decoder input from generated tokens so far
dec_input = torch.tensor(
[generated], dtype=torch.long, device=device
) # (1, t)
# Decode
logits = self.decoder(dec_input, encoder_out) # (1, t, vocab)
next_logits = logits[0, -1, :] # (vocab,)
# Repetition penalty: reduce logits for tokens already generated
if repetition_penalty != 1.0:
for prev_token in set(generated):
if next_logits[prev_token] > 0:
next_logits[prev_token] /= repetition_penalty
else:
next_logits[prev_token] *= repetition_penalty
# No-repeat n-gram blocking
if no_repeat_ngram > 0 and len(generated) >= no_repeat_ngram:
# Check if any token would create a repeated n-gram
ngram_prefix = generated[-(no_repeat_ngram - 1):]
for i in range(len(generated) - no_repeat_ngram + 1):
if generated[i:i + no_repeat_ngram - 1] == ngram_prefix:
blocked_token = generated[i + no_repeat_ngram - 1]
next_logits[blocked_token] = float("-inf")
next_token = next_logits.argmax().item()
generated.append(next_token)
if next_token == eos_id:
break
# Return without the initial SOS
return torch.tensor(generated[1:], dtype=torch.long)
|