File size: 6,402 Bytes
a7452cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
"""
Grassmann Seq2Seq Model — Fully Attention-Free Encoder-Decoder.

Audio → Grassmann Encoder → Grassmann Decoder → Text

Every sequence interaction in this model happens through Plücker
coordinates on the Grassmann manifold Gr(2, r). There are zero
attention matrices anywhere in the architecture:

  Encoder self-mixing:  Plücker(z_t, z_{t+Δ})  — local causal windows
  Decoder self-mixing:  Plücker(z_t, z_{t+Δ})  — causal autoregressive
  Decoder cross-mixing: Plücker(z_dec, z_enc)   — monotonic alignment windows

Trained with teacher forcing and cross-entropy loss.
"""

import torch
import torch.nn as nn

from .config import GrassmannConfig
from .grassmann_model import ConvFrontend, PositionalEncoding
from .grassmann_layer import CausalGrassmannLayer
from .grassmann_decoder import GrassmannDecoder


class GrassmannEncoder(nn.Module):
    """Audio encoder: mel spectrogram → Grassmann layers → encoder states."""

    def __init__(self, config: GrassmannConfig):
        super().__init__()
        self.frontend = ConvFrontend(config.n_mels, config.d_model)
        self.pos_enc = PositionalEncoding(
            config.d_model, config.max_seq_len, config.dropout
        )
        self.layers = nn.ModuleList([
            CausalGrassmannLayer(config, i) for i in range(config.n_layers)
        ])

    def forward(self, mel: torch.Tensor) -> torch.Tensor:
        """
        Args:
            mel: (batch, n_mels, time) - mel spectrogram

        Returns:
            (batch, T_enc, d_model) - encoder hidden states
        """
        x = self.frontend(mel)
        x = self.pos_enc(x)
        for layer in self.layers:
            x = layer(x)
        return x


class GrassmannSeq2SeqModel(nn.Module):
    """
    Fully attention-free encoder-decoder for audio transcription.

    Architecture:
      Encoder: ConvFrontend → PosEnc → N × CausalGrassmannLayer
      Decoder: TokenEmbed → PosEmbed → M × (CausalGrassmann + CrossGrassmann + FFN)
      Output:  Linear → vocab logits

    All interactions use Plücker coordinates. Zero attention matrices.
    """

    # Special token IDs
    SOS_ID = 0  # Start of sequence (overloads blank/pad)
    EOS_ID = 0  # End of sequence (same as SOS for simplicity)
    PAD_ID = 0  # Padding

    def __init__(self, config: GrassmannConfig):
        super().__init__()
        self.config = config
        self.encoder = GrassmannEncoder(config)
        self.decoder = GrassmannDecoder(config)

    def forward(
        self,
        mel: torch.Tensor,
        target_tokens: torch.Tensor,
    ) -> torch.Tensor:
        """
        Teacher-forced forward pass.

        Args:
            mel:           (batch, n_mels, time) - input mel spectrogram
            target_tokens: (batch, T_target) - target token indices

        Returns:
            logits: (batch, T_target, vocab_size)
        """
        # Encode audio
        encoder_out = self.encoder(mel)  # (B, T_enc, d)

        # Decode with teacher forcing
        # Input to decoder is target shifted right (prepend SOS)
        dec_input = self._shift_right(target_tokens)
        logits = self.decoder(dec_input, encoder_out)

        return logits

    def _shift_right(self, tokens: torch.Tensor) -> torch.Tensor:
        """Shift tokens right and prepend SOS token for teacher forcing."""
        sos = torch.full(
            (tokens.shape[0], 1), self.SOS_ID,
            dtype=tokens.dtype, device=tokens.device
        )
        # Prepend SOS, drop last token
        return torch.cat([sos, tokens[:, :-1]], dim=1)

    def get_encoder_output_lengths(self, input_lengths: torch.Tensor) -> torch.Tensor:
        """Calculate encoder output lengths after conv downsampling."""
        lengths = input_lengths
        for _ in range(2):  # 2 conv layers with stride 2
            lengths = (lengths + 1) // 2
        return lengths

    @torch.no_grad()
    def greedy_decode(
        self,
        mel: torch.Tensor,
        max_len: int = 200,
        eos_id: int = 0,
        repetition_penalty: float = 1.5,
        no_repeat_ngram: int = 3,
    ) -> torch.Tensor:
        """
        Autoregressive greedy decoding with repetition suppression.

        Args:
            mel:                (1, n_mels, time) - single mel spectrogram
            max_len:            maximum output length
            eos_id:             end-of-sequence token ID
            repetition_penalty: penalize tokens that already appeared (>1.0 to suppress)
            no_repeat_ngram:    block any n-gram that already occurred

        Returns:
            (seq_len,) - predicted token indices
        """
        self.eval()
        device = mel.device

        # Encode once
        encoder_out = self.encoder(mel)  # (1, T_enc, d)

        # Start with SOS token
        generated = [self.SOS_ID]

        for _ in range(max_len):
            # Build decoder input from generated tokens so far
            dec_input = torch.tensor(
                [generated], dtype=torch.long, device=device
            )  # (1, t)

            # Decode
            logits = self.decoder(dec_input, encoder_out)  # (1, t, vocab)
            next_logits = logits[0, -1, :]  # (vocab,)

            # Repetition penalty: reduce logits for tokens already generated
            if repetition_penalty != 1.0:
                for prev_token in set(generated):
                    if next_logits[prev_token] > 0:
                        next_logits[prev_token] /= repetition_penalty
                    else:
                        next_logits[prev_token] *= repetition_penalty

            # No-repeat n-gram blocking
            if no_repeat_ngram > 0 and len(generated) >= no_repeat_ngram:
                # Check if any token would create a repeated n-gram
                ngram_prefix = generated[-(no_repeat_ngram - 1):]
                for i in range(len(generated) - no_repeat_ngram + 1):
                    if generated[i:i + no_repeat_ngram - 1] == ngram_prefix:
                        blocked_token = generated[i + no_repeat_ngram - 1]
                        next_logits[blocked_token] = float("-inf")

            next_token = next_logits.argmax().item()
            generated.append(next_token)

            if next_token == eos_id:
                break

        # Return without the initial SOS
        return torch.tensor(generated[1:], dtype=torch.long)