File size: 4,420 Bytes
d7013e6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
"""
Grassmann Cross-Mixing Layer.

Replaces cross-attention in the decoder with Grassmann geometry.
Instead of Q*K^T dot products between decoder and encoder states,
we form Plücker coordinates between decoder positions and encoder
positions within a local monotonic window.

This is the key novel component — cross-sequence interaction via
subspace geometry on Gr(2, r) rather than attention weights.
"""

import torch
import torch.nn as nn

from .grassmann_layer import PluckerEncoder
from .config import GrassmannConfig


class GrassmannCrossLayer(nn.Module):
    """
    Cross-mixing between decoder and encoder via Grassmann/Plücker geometry.

    For each decoder position t, estimates an alignment position in the
    encoder sequence, then computes Plücker features between the decoder
    state and encoder states in a window around that alignment.

    This replaces cross-attention entirely — no Q*K^T, no softmax,
    just subspace geometry on a manifold.
    """

    def __init__(self, config: GrassmannConfig):
        super().__init__()
        d = config.d_model
        r = config.r_reduced
        plucker_dim = config.plucker_dim

        # Separate reduction projections for decoder and encoder
        self.W_red_dec = nn.Linear(d, r)
        self.W_red_enc = nn.Linear(d, r)

        # Plücker encoder (shared geometry)
        self.plucker = PluckerEncoder(r, eps=config.plucker_eps)

        # Project Plücker features back to model dim
        self.W_plu = nn.Linear(plucker_dim, d)

        # Gated fusion: blend decoder state with cross-mixed features
        self.W_gate = nn.Linear(2 * d, d)

        # Layer norm
        self.norm = nn.LayerNorm(d)
        self.dropout = nn.Dropout(config.dropout)

        # Cross-mixing window: how many encoder positions to look at
        # per decoder position (on each side of the alignment point)
        self.cross_window = config.cross_window

    def _estimate_alignment(
        self, dec_len: int, enc_len: int, device: torch.device
    ) -> torch.Tensor:
        """
        Estimate monotonic alignment: decoder position t maps to
        encoder position t * (enc_len / dec_len).

        Returns integer encoder indices for each decoder position.
        """
        # Linear interpolation: dec pos 0..T_dec maps to enc pos 0..T_enc
        dec_positions = torch.arange(dec_len, device=device, dtype=torch.float)
        scale = enc_len / max(dec_len, 1)
        enc_positions = (dec_positions * scale).long().clamp(0, enc_len - 1)
        return enc_positions

    def forward(
        self, h_dec: torch.Tensor, h_enc: torch.Tensor
    ) -> torch.Tensor:
        """
        Args:
            h_dec: (batch, T_dec, d_model) - decoder hidden states
            h_enc: (batch, T_enc, d_model) - encoder hidden states

        Returns:
            h_out: (batch, T_dec, d_model) - cross-mixed decoder states
        """
        B, T_dec, D = h_dec.shape
        T_enc = h_enc.shape[1]

        # Reduce both to Grassmann space
        z_dec = self.W_red_dec(h_dec)   # (B, T_dec, r)
        z_enc = self.W_red_enc(h_enc)   # (B, T_enc, r)

        # Estimate alignment positions
        align = self._estimate_alignment(T_dec, T_enc, h_dec.device)  # (T_dec,)

        # For each decoder position, gather encoder states in a window
        # around the alignment point and compute Plücker features
        w = self.cross_window
        g_accum = torch.zeros_like(h_dec)  # (B, T_dec, D)
        count = 0

        for offset in range(-w, w + 1):
            # Encoder indices for this offset
            enc_idx = (align + offset).clamp(0, T_enc - 1)  # (T_dec,)

            # Gather encoder states at these positions
            enc_gathered = z_enc[:, enc_idx, :]  # (B, T_dec, r)

            # Plücker encoding between decoder and encoder states
            p = self.plucker(z_dec, enc_gathered)  # (B, T_dec, plucker_dim)

            # Project back
            g = self.W_plu(p)  # (B, T_dec, D)
            g_accum += g
            count += 1

        g_cross = g_accum / max(count, 1)

        # Gated fusion
        u = torch.cat([h_dec, g_cross], dim=-1)  # (B, T_dec, 2D)
        alpha = torch.sigmoid(self.W_gate(u))      # (B, T_dec, D)
        h_mixed = alpha * h_dec + (1 - alpha) * g_cross

        # Residual + layer norm + dropout
        h_out = self.norm(h_dec + self.dropout(h_mixed))

        return h_out