File size: 6,030 Bytes
d7013e6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
"""
Full Grassmann Transcription Model.

Audio β†’ Mel Spectrogram features β†’ Grassmann Layers β†’ CTC Output

Architecture:
  1. Conv frontend: projects mel features into d_model
  2. Positional encoding
  3. N x CausalGrassmannLayer
  4. Linear CTC head β†’ vocab_size
"""

import math

import torch
import torch.nn as nn
import torch.nn.functional as F

from .config import GrassmannConfig
from .grassmann_layer import CausalGrassmannLayer
from .ablation_layer import AblationLayer
from .conformer_block import GrassmannConformerBlock, AblationConformerBlock


class ConvFrontend(nn.Module):
    """Conv1d feature extractor for mel spectrograms (legacy)."""

    def __init__(self, n_mels: int, d_model: int):
        super().__init__()
        self.conv = nn.Sequential(
            nn.Conv1d(n_mels, d_model // 2, kernel_size=3, stride=2, padding=1),
            nn.GELU(),
            nn.Conv1d(d_model // 2, d_model, kernel_size=3, stride=2, padding=1),
            nn.GELU(),
        )

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        out = self.conv(x)  # (B, d_model, T')
        return out.transpose(1, 2)  # (B, T', d_model)


class Conv2dSubsampling(nn.Module):
    """
    Conv2d subsampling from Conformer (Gulati et al. 2020).

    Two 3x3 stride-2 convolutions: 10ms fbank frames β†’ 40ms encoder frames.
    Standard ASR frontend used by ESPnet, WeNet, etc.
    """

    def __init__(self, n_mels: int, channels: int, d_model: int):
        super().__init__()
        self.conv1 = nn.Conv2d(1, channels, kernel_size=3, stride=2)
        self.conv2 = nn.Conv2d(channels, channels, kernel_size=3, stride=2)
        # After two 3x3 stride-2 convs: freq dim = ((n_mels - 1) // 2 - 1) // 2
        freq_out = ((n_mels - 1) // 2 - 1) // 2
        self.out = nn.Linear(channels * freq_out, d_model)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Args:
            x: (batch, n_mels, time) - mel spectrogram

        Returns:
            (batch, time', d_model)
        """
        # Reshape: (B, n_mels, T) β†’ (B, 1, T, n_mels) for Conv2d
        x = x.transpose(1, 2).unsqueeze(1)  # (B, 1, T, n_mels)
        x = F.relu(self.conv1(x))
        x = F.relu(self.conv2(x))
        b, c, t, f = x.shape
        x = x.transpose(1, 2).reshape(b, t, c * f)  # (B, T', C*F)
        return self.out(x)  # (B, T', d_model)


class PositionalEncoding(nn.Module):
    """Sinusoidal positional encoding."""

    def __init__(self, d_model: int, max_len: int = 5000, dropout: float = 0.1):
        super().__init__()
        self.dropout = nn.Dropout(dropout)

        pe = torch.zeros(max_len, d_model)
        position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
        div_term = torch.exp(
            torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)
        )
        pe[:, 0::2] = torch.sin(position * div_term)
        pe[:, 1::2] = torch.cos(position * div_term)
        self.register_buffer("pe", pe.unsqueeze(0))  # (1, max_len, d_model)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = x + self.pe[:, : x.size(1)]
        return self.dropout(x)


class GrassmannTranscriptionModel(nn.Module):
    """
    Full model: Audio features β†’ Grassmann layers β†’ CTC output.

    Supports two encoder modes:
      - Plain: CausalGrassmannLayer stack (original)
      - Conformer: GrassmannConformerBlock stack (with ConvModule + Macaron FFN)

    And two frontend modes:
      - Conv1d (legacy)
      - Conv2d subsampling (proper ASR, from Conformer)
    """

    def __init__(self, config: GrassmannConfig):
        super().__init__()
        self.config = config

        # Frontend: mel β†’ d_model
        if config.use_conformer:
            self.frontend = Conv2dSubsampling(
                config.n_mels, config.conv_channels, config.d_model
            )
        else:
            self.frontend = ConvFrontend(config.n_mels, config.d_model)

        # Positional encoding
        self.pos_enc = PositionalEncoding(
            config.d_model, config.max_seq_len, config.dropout
        )

        # Encoder layers
        ablation = getattr(config, "use_ablation", False)
        if config.use_conformer:
            block_cls = AblationConformerBlock if ablation else GrassmannConformerBlock
            self.layers = nn.ModuleList(
                [block_cls(config) for _ in range(config.n_layers)]
            )
        else:
            layer_cls = AblationLayer if ablation else CausalGrassmannLayer
            self.layers = nn.ModuleList(
                [layer_cls(config, i) for i in range(config.n_layers)]
            )

        # CTC output head
        self.output_proj = nn.Linear(config.d_model, config.vocab_size)

    def forward(self, mel: torch.Tensor) -> torch.Tensor:
        """
        Args:
            mel: (batch, n_mels, time) - mel spectrogram

        Returns:
            log_probs: (time', batch, vocab_size) - CTC-formatted log probs
        """
        # Conv frontend
        x = self.frontend(mel)  # (B, T', d_model)

        # Positional encoding
        x = self.pos_enc(x)

        # Grassmann layers
        for layer in self.layers:
            x = layer(x)

        # CTC head
        logits = self.output_proj(x)  # (B, T', vocab_size)
        log_probs = logits.log_softmax(dim=-1)

        # CTC expects (T, B, C)
        return log_probs.transpose(0, 1)

    def get_output_lengths(self, input_lengths: torch.Tensor) -> torch.Tensor:
        """Calculate output sequence lengths after conv downsampling."""
        lengths = input_lengths
        if self.config.use_conformer:
            # Conv2d subsampling: two 3x3 stride-2 convs (no padding)
            # Each conv: out = (in - 3) // 2 + 1 = (in - 1) // 2
            for _ in range(2):
                lengths = (lengths - 1) // 2
        else:
            # Conv1d frontend: two stride-2 convs with padding=1
            for _ in range(2):
                lengths = (lengths + 1) // 2
        return lengths