Spaces:
Running on T4
Running on T4
File size: 3,993 Bytes
a7452cc | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 | """
Grassmann Decoder.
A fully attention-free autoregressive decoder using:
1. Causal Grassmann self-mixing (same as encoder)
2. Grassmann cross-mixing (replaces cross-attention)
3. Position-wise FFN
Each decoder block: CausalSelfMix → CrossMix → FFN
Stacked M times, with token embeddings and output projection.
"""
import math
import torch
import torch.nn as nn
from .config import GrassmannConfig
from .grassmann_layer import CausalGrassmannLayer
from .grassmann_cross_layer import GrassmannCrossLayer
class GrassmannDecoderBlock(nn.Module):
"""
One decoder block: self-mixing → cross-mixing → FFN.
The self-mixing uses CausalGrassmannLayer (causal, decoder-to-decoder).
The cross-mixing uses GrassmannCrossLayer (decoder-to-encoder).
The FFN is already included in CausalGrassmannLayer, so we only add
the cross-mixing step between self-mixing and the FFN residual.
"""
def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
super().__init__()
d = config.d_model
# Causal self-mixing (includes FFN)
self.self_mix = CausalGrassmannLayer(config, layer_idx)
# Grassmann cross-mixing with encoder
self.cross_mix = GrassmannCrossLayer(config)
# Additional FFN after cross-mixing
self.ffn = nn.Sequential(
nn.Linear(d, config.d_ff),
nn.GELU(),
nn.Dropout(config.dropout),
nn.Linear(config.d_ff, d),
nn.Dropout(config.dropout),
)
self.norm_ffn = nn.LayerNorm(d)
def forward(self, h_dec: torch.Tensor, h_enc: torch.Tensor) -> torch.Tensor:
"""
Args:
h_dec: (batch, T_dec, d_model) - decoder states
h_enc: (batch, T_enc, d_model) - encoder output
Returns:
(batch, T_dec, d_model) - updated decoder states
"""
# Self-mixing (causal, within decoder sequence)
h = self.self_mix(h_dec)
# Cross-mixing (decoder ↔ encoder via Grassmann geometry)
h = self.cross_mix(h, h_enc)
# FFN with residual
h_out = self.norm_ffn(h + self.ffn(h))
return h_out
class GrassmannDecoder(nn.Module):
"""
Full Grassmann decoder: token embeddings → M decoder blocks → output logits.
Fully attention-free. All sequence interactions happen through
Plücker coordinates on Gr(2, r).
"""
def __init__(self, config: GrassmannConfig):
super().__init__()
self.config = config
# Token embeddings
self.token_embed = nn.Embedding(config.vocab_size, config.d_model)
self.pos_embed = nn.Embedding(config.max_seq_len, config.d_model)
self.embed_dropout = nn.Dropout(config.dropout)
self.embed_scale = math.sqrt(config.d_model)
# Decoder blocks
self.blocks = nn.ModuleList([
GrassmannDecoderBlock(config, i)
for i in range(config.n_decoder_layers)
])
# Output projection to vocab
self.output_norm = nn.LayerNorm(config.d_model)
self.output_proj = nn.Linear(config.d_model, config.vocab_size)
def forward(
self,
tokens: torch.Tensor,
encoder_out: torch.Tensor,
) -> torch.Tensor:
"""
Args:
tokens: (batch, T_dec) - input token indices (shifted right)
encoder_out: (batch, T_enc, d_model) - encoder hidden states
Returns:
logits: (batch, T_dec, vocab_size)
"""
B, T = tokens.shape
# Token + positional embeddings
positions = torch.arange(T, device=tokens.device).unsqueeze(0)
h = self.token_embed(tokens) * self.embed_scale + self.pos_embed(positions)
h = self.embed_dropout(h)
# Decoder blocks
for block in self.blocks:
h = block(h, encoder_out)
# Output
h = self.output_norm(h)
logits = self.output_proj(h)
return logits
|