File size: 6,314 Bytes
a7452cc
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
"""
Causal Grassmann Mixing Layer.

Implements the core architecture from "Attention Is Not What You Need"
(arXiv:2512.19428). Replaces self-attention with:

1. Linear reduction:       h_t ∈ R^d  →  z_t ∈ R^r
2. Multi-scale pairing:    (z_t, z_{t+Δ}) for Δ ∈ W
3. Plücker encoding:       p_{ij} = z_{t,i} * z_{t+Δ,j} - z_{t,j} * z_{t+Δ,i}
4. Projection back:        g_t = W_plu @ p_hat + b_plu
5. Gated fusion:           h_mix = α ⊙ h + (1-α) ⊙ g
6. Feed-forward block:     standard FFN with GELU
"""

import torch
import torch.nn as nn
import torch.nn.functional as F

from .config import GrassmannConfig


class PluckerEncoder(nn.Module):
    """Computes Plücker coordinates for pairs of reduced vectors."""

    def __init__(self, r: int, eps: float = 1e-8):
        super().__init__()
        self.r = r
        self.eps = eps
        # Precompute index pairs (i, j) where i < j
        idx_i, idx_j = [], []
        for i in range(r):
            for j in range(i + 1, r):
                idx_i.append(i)
                idx_j.append(j)
        self.register_buffer("idx_i", torch.tensor(idx_i, dtype=torch.long))
        self.register_buffer("idx_j", torch.tensor(idx_j, dtype=torch.long))

    def forward(self, z_t: torch.Tensor, z_delta: torch.Tensor) -> torch.Tensor:
        """
        Compute normalized Plücker coordinates.

        Args:
            z_t:     (batch, seq, r) - reduced states at position t
            z_delta: (batch, seq, r) - reduced states at position t+Δ

        Returns:
            (batch, seq, C(r,2)) - normalized Plücker vectors
        """
        # p_{ij} = z_{t,i} * z_{Δ,j} - z_{t,j} * z_{Δ,i}
        p = (
            z_t[..., self.idx_i] * z_delta[..., self.idx_j]
            - z_t[..., self.idx_j] * z_delta[..., self.idx_i]
        )
        # Normalize for numerical stability
        norm = p.norm(dim=-1, keepdim=True).clamp(min=self.eps)
        return p / norm


class GrassmannMixer(nn.Module):
    """
    Grassmann/Plücker mixer — the core sequence interaction.

    Supports:
      - Causal (one-sided) or bidirectional (two-sided) mixing
      - Mean-first (anchored) form: 3.5x faster, mathematically exact
      - Standard (as-written) form: project then average

    Two-sided mixing is appropriate for CTC/encoder mode since speech
    isn't generated left-to-right. Antisymmetry of Plücker coordinates
    naturally distinguishes past from future even in two-sided mode.
    """

    def __init__(self, d: int, r: int, offsets: list[int],
                 two_sided: bool = False, eps: float = 1e-8):
        super().__init__()
        self.offsets = offsets
        self.two_sided = two_sided
        self.eps = eps
        plucker_dim = r * (r - 1) // 2

        self.W_red = nn.Linear(d, r)
        self.plucker = PluckerEncoder(r, eps=eps)
        self.W_plu = nn.Linear(plucker_dim, d)

    def forward(self, h: torch.Tensor) -> torch.Tensor:
        """
        Mean-first (anchored) form: average partners before projection.
        Mathematically equivalent to as-written, but ~3.5x faster on GPU
        because it does one W_plu matmul instead of one per offset.
        """
        B, L, D = h.shape
        z = self.W_red(h)  # (B, L, r)

        # Accumulate weighted partner vectors, then compute Plücker once
        partner = torch.zeros_like(z)  # (B, L, r)
        count = z.new_zeros(1, L, 1)

        for delta in self.offsets:
            if delta >= L:
                continue
            earlier = z[:, :L - delta, :]
            later = z[:, delta:, :]

            # Compute norm for weighting (from dot products, numerically stable)
            p = self.plucker(earlier, later)
            w = p.norm(dim=-1, keepdim=True).clamp(min=self.eps).reciprocal()

            # Assign to later position (causal: t+Δ sees t)
            partner[:, delta:] += w * earlier
            count[:, delta:] += 1

            if self.two_sided:
                # Also let earlier position see later (bidirectional)
                # Plücker antisymmetry (flip sign) distinguishes direction
                partner[:, :L - delta] += -w * later
                count[:, :L - delta] += 1

        # Average partners, compute Plücker features, project
        count = count.clamp(min=1)
        avg_partner = partner / count
        # Compute final Plücker between each position and its avg partner
        p_final = self.plucker(z, avg_partner)
        return self.W_plu(p_final)


class CausalGrassmannLayer(nn.Module):
    """
    One layer of the Causal Grassmann Transformer.

    Replaces the self-attention block with Grassmann/Plücker mixing,
    followed by the standard position-wise FFN.

    Supports two_sided mode for encoder/CTC use cases.
    """

    def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
        super().__init__()
        d = config.d_model
        r = config.r_reduced
        two_sided = getattr(config, "two_sided", False)

        # Grassmann mixer
        self.mixer = GrassmannMixer(
            d, r, config.window_offsets,
            two_sided=two_sided, eps=config.plucker_eps,
        )

        # Gated fusion
        self.W_gate = nn.Linear(2 * d, d)

        # Layer norm after mixing
        self.norm_mix = nn.LayerNorm(d)
        self.dropout_mix = nn.Dropout(config.dropout)

        # Feed-forward block
        self.ffn = nn.Sequential(
            nn.Linear(d, config.d_ff),
            nn.GELU(),
            nn.Dropout(config.dropout),
            nn.Linear(config.d_ff, d),
            nn.Dropout(config.dropout),
        )
        self.norm_ffn = nn.LayerNorm(d)

    def forward(self, h: torch.Tensor) -> torch.Tensor:
        """
        Args:
            h: (batch, seq_len, d_model)
        Returns:
            h': (batch, seq_len, d_model)
        """
        # Grassmann mixing
        g = self.mixer(h)  # (B, L, D)

        # Gated fusion: (1-α)(g-h) as residual (Dhruv's variant)
        u = torch.cat([h, g], dim=-1)
        alpha = torch.sigmoid(self.W_gate(u))
        h_mix = h + (1 - alpha) * (g - h)

        # Layer norm + dropout + residual
        h_hat = self.norm_mix(h + self.dropout_mix(h_mix))

        # Feed-forward with residual
        h_out = self.norm_ffn(h_hat + self.ffn(h_hat))

        return h_out