File size: 3,993 Bytes
d7013e6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
"""
Grassmann Decoder.

A fully attention-free autoregressive decoder using:
  1. Causal Grassmann self-mixing (same as encoder)
  2. Grassmann cross-mixing (replaces cross-attention)
  3. Position-wise FFN

Each decoder block: CausalSelfMix → CrossMix → FFN
Stacked M times, with token embeddings and output projection.
"""

import math

import torch
import torch.nn as nn

from .config import GrassmannConfig
from .grassmann_layer import CausalGrassmannLayer
from .grassmann_cross_layer import GrassmannCrossLayer


class GrassmannDecoderBlock(nn.Module):
    """
    One decoder block: self-mixing → cross-mixing → FFN.

    The self-mixing uses CausalGrassmannLayer (causal, decoder-to-decoder).
    The cross-mixing uses GrassmannCrossLayer (decoder-to-encoder).
    The FFN is already included in CausalGrassmannLayer, so we only add
    the cross-mixing step between self-mixing and the FFN residual.
    """

    def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
        super().__init__()
        d = config.d_model

        # Causal self-mixing (includes FFN)
        self.self_mix = CausalGrassmannLayer(config, layer_idx)

        # Grassmann cross-mixing with encoder
        self.cross_mix = GrassmannCrossLayer(config)

        # Additional FFN after cross-mixing
        self.ffn = nn.Sequential(
            nn.Linear(d, config.d_ff),
            nn.GELU(),
            nn.Dropout(config.dropout),
            nn.Linear(config.d_ff, d),
            nn.Dropout(config.dropout),
        )
        self.norm_ffn = nn.LayerNorm(d)

    def forward(self, h_dec: torch.Tensor, h_enc: torch.Tensor) -> torch.Tensor:
        """
        Args:
            h_dec: (batch, T_dec, d_model) - decoder states
            h_enc: (batch, T_enc, d_model) - encoder output

        Returns:
            (batch, T_dec, d_model) - updated decoder states
        """
        # Self-mixing (causal, within decoder sequence)
        h = self.self_mix(h_dec)

        # Cross-mixing (decoder ↔ encoder via Grassmann geometry)
        h = self.cross_mix(h, h_enc)

        # FFN with residual
        h_out = self.norm_ffn(h + self.ffn(h))

        return h_out


class GrassmannDecoder(nn.Module):
    """
    Full Grassmann decoder: token embeddings → M decoder blocks → output logits.

    Fully attention-free. All sequence interactions happen through
    Plücker coordinates on Gr(2, r).
    """

    def __init__(self, config: GrassmannConfig):
        super().__init__()
        self.config = config

        # Token embeddings
        self.token_embed = nn.Embedding(config.vocab_size, config.d_model)
        self.pos_embed = nn.Embedding(config.max_seq_len, config.d_model)
        self.embed_dropout = nn.Dropout(config.dropout)
        self.embed_scale = math.sqrt(config.d_model)

        # Decoder blocks
        self.blocks = nn.ModuleList([
            GrassmannDecoderBlock(config, i)
            for i in range(config.n_decoder_layers)
        ])

        # Output projection to vocab
        self.output_norm = nn.LayerNorm(config.d_model)
        self.output_proj = nn.Linear(config.d_model, config.vocab_size)

    def forward(
        self,
        tokens: torch.Tensor,
        encoder_out: torch.Tensor,
    ) -> torch.Tensor:
        """
        Args:
            tokens:      (batch, T_dec) - input token indices (shifted right)
            encoder_out: (batch, T_enc, d_model) - encoder hidden states

        Returns:
            logits: (batch, T_dec, vocab_size)
        """
        B, T = tokens.shape

        # Token + positional embeddings
        positions = torch.arange(T, device=tokens.device).unsqueeze(0)
        h = self.token_embed(tokens) * self.embed_scale + self.pos_embed(positions)
        h = self.embed_dropout(h)

        # Decoder blocks
        for block in self.blocks:
            h = block(h, encoder_out)

        # Output
        h = self.output_norm(h)
        logits = self.output_proj(h)

        return logits