File size: 4,768 Bytes
d7013e6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
"""
Conformer Block with Grassmann Mixer.

Macaron-style block from Gulati et al. 2020 (arXiv 2005.08100)
with the self-attention slot replaced by a Grassmann mixer:

  x β†’ +0.5*FFN β†’ +Mixer β†’ +Conv β†’ +0.5*FFN β†’ LayerNorm

This gives the best of both worlds:
  - Grassmann mixer for sequence-level subspace geometry
  - ConvModule for local temporal pattern capture
  - Macaron FFN sandwich for nonlinear capacity
"""

import torch
import torch.nn as nn

from .config import GrassmannConfig
from .conv_module import ConvModule
from .grassmann_layer import GrassmannMixer
from .ablation_layer import AblationMixer


def _macaron_ffn(d: int, d_ff: int, dropout: float) -> nn.Sequential:
    """Conformer-style FFN: LayerNorm β†’ Linear β†’ SiLU β†’ Dropout β†’ Linear β†’ Dropout."""
    return nn.Sequential(
        nn.LayerNorm(d),
        nn.Linear(d, d_ff),
        nn.SiLU(),
        nn.Dropout(dropout),
        nn.Linear(d_ff, d),
        nn.Dropout(dropout),
    )


class GatedGrassmannMixer(nn.Module):
    """
    Two-sided Grassmann mix with sigmoid gate as a residual branch.
    (1 - Ξ±)(g - u) formulation from Dhruv's implementation.
    """

    def __init__(self, d: int, r: int, offsets: list[int], eps: float = 1e-8):
        super().__init__()
        self.mixer = GrassmannMixer(d, r, offsets, two_sided=True, eps=eps)
        self.gate = nn.Linear(2 * d, d)

    def forward(self, u: torch.Tensor) -> torch.Tensor:
        g = self.mixer(u)
        alpha = torch.sigmoid(self.gate(torch.cat([u, g], dim=-1)))
        return (1 - alpha) * (g - u)


class GatedAblationMixer(nn.Module):
    """
    Ablation baseline: symmetric-product mixer with the same gate.
    Drop-in replacement for GatedGrassmannMixer.
    """

    def __init__(self, d: int, r: int, offsets: list[int], eps: float = 1e-8):
        super().__init__()
        self.mixer = AblationMixer(d, r, offsets, two_sided=True, eps=eps)
        self.gate = nn.Linear(2 * d, d)

    def forward(self, u: torch.Tensor) -> torch.Tensor:
        g = self.mixer(u)
        alpha = torch.sigmoid(self.gate(torch.cat([u, g], dim=-1)))
        return (1 - alpha) * (g - u)


class GrassmannConformerBlock(nn.Module):
    """
    Conformer block with Grassmann mixer replacing self-attention.

    Architecture (Macaron block):
      x β†’ +0.5*FFN1 β†’ +GatedGrassmannMixer β†’ +ConvModule β†’ +0.5*FFN2 β†’ LayerNorm
    """

    def __init__(self, config: GrassmannConfig):
        super().__init__()
        d = config.d_model
        d_ff = config.d_ff
        r = config.r_reduced
        dropout = config.dropout
        kernel = getattr(config, "conv_kernel_size", 31)
        offsets = config.window_offsets

        # Macaron FFN sandwich
        self.ff1 = _macaron_ffn(d, d_ff, dropout)
        self.ff2 = _macaron_ffn(d, d_ff, dropout)

        # Grassmann mixer (replaces self-attention)
        self.ln_mix = nn.LayerNorm(d)
        self.mixer = GatedGrassmannMixer(d, r, offsets, eps=config.plucker_eps)
        self.drop = nn.Dropout(dropout)

        # ConvModule for local patterns
        self.conv = ConvModule(d, kernel, dropout)

        # Final LayerNorm
        self.ln_out = nn.LayerNorm(d)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        """
        Args:
            x: (batch, seq_len, d_model)
        Returns:
            (batch, seq_len, d_model)
        """
        # Half-step FFN 1
        x = x + 0.5 * self.ff1(x)
        # Grassmann mixer (with gate as residual)
        x = x + self.drop(self.mixer(self.ln_mix(x)))
        # ConvModule
        x = x + self.conv(x)
        # Half-step FFN 2 + final norm
        return self.ln_out(x + 0.5 * self.ff2(x))


class AblationConformerBlock(nn.Module):
    """
    Conformer block with symmetric-product mixer (ablation baseline).
    Same Macaron FFN + ConvModule, just swaps Grassmann for symmetric products.
    """

    def __init__(self, config: GrassmannConfig):
        super().__init__()
        d = config.d_model
        d_ff = config.d_ff
        r = config.r_reduced
        dropout = config.dropout
        kernel = getattr(config, "conv_kernel_size", 31)
        offsets = config.window_offsets

        self.ff1 = _macaron_ffn(d, d_ff, dropout)
        self.ff2 = _macaron_ffn(d, d_ff, dropout)

        self.ln_mix = nn.LayerNorm(d)
        self.mixer = GatedAblationMixer(d, r, offsets, eps=config.plucker_eps)
        self.drop = nn.Dropout(dropout)

        self.conv = ConvModule(d, kernel, dropout)
        self.ln_out = nn.LayerNorm(d)

    def forward(self, x: torch.Tensor) -> torch.Tensor:
        x = x + 0.5 * self.ff1(x)
        x = x + self.drop(self.mixer(self.ln_mix(x)))
        x = x + self.conv(x)
        return self.ln_out(x + 0.5 * self.ff2(x))