grassmann-cache / src /models /grassmann_decoder.py
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache — ProcessPoolExecutor hangs on cpu-basic
d7013e6
Raw History Blame Contribute Delete
3.99 kB
"""
Grassmann Decoder.
A fully attention-free autoregressive decoder using:
1. Causal Grassmann self-mixing (same as encoder)
2. Grassmann cross-mixing (replaces cross-attention)
3. Position-wise FFN
Each decoder block: CausalSelfMix → CrossMix → FFN
Stacked M times, with token embeddings and output projection.
"""
import math
import torch
import torch.nn as nn
from .config import GrassmannConfig
from .grassmann_layer import CausalGrassmannLayer
from .grassmann_cross_layer import GrassmannCrossLayer
class GrassmannDecoderBlock(nn.Module):
"""
One decoder block: self-mixing → cross-mixing → FFN.
The self-mixing uses CausalGrassmannLayer (causal, decoder-to-decoder).
The cross-mixing uses GrassmannCrossLayer (decoder-to-encoder).
The FFN is already included in CausalGrassmannLayer, so we only add
the cross-mixing step between self-mixing and the FFN residual.
"""
def __init__(self, config: GrassmannConfig, layer_idx: int = 0):
super().__init__()
d = config.d_model
# Causal self-mixing (includes FFN)
self.self_mix = CausalGrassmannLayer(config, layer_idx)
# Grassmann cross-mixing with encoder
self.cross_mix = GrassmannCrossLayer(config)
# Additional FFN after cross-mixing
self.ffn = nn.Sequential(
nn.Linear(d, config.d_ff),
nn.GELU(),
nn.Dropout(config.dropout),
nn.Linear(config.d_ff, d),
nn.Dropout(config.dropout),
)
self.norm_ffn = nn.LayerNorm(d)
def forward(self, h_dec: torch.Tensor, h_enc: torch.Tensor) -> torch.Tensor:
"""
Args:
h_dec: (batch, T_dec, d_model) - decoder states
h_enc: (batch, T_enc, d_model) - encoder output
Returns:
(batch, T_dec, d_model) - updated decoder states
"""
# Self-mixing (causal, within decoder sequence)
h = self.self_mix(h_dec)
# Cross-mixing (decoder ↔ encoder via Grassmann geometry)
h = self.cross_mix(h, h_enc)
# FFN with residual
h_out = self.norm_ffn(h + self.ffn(h))
return h_out
class GrassmannDecoder(nn.Module):
"""
Full Grassmann decoder: token embeddings → M decoder blocks → output logits.
Fully attention-free. All sequence interactions happen through
Plücker coordinates on Gr(2, r).
"""
def __init__(self, config: GrassmannConfig):
super().__init__()
self.config = config
# Token embeddings
self.token_embed = nn.Embedding(config.vocab_size, config.d_model)
self.pos_embed = nn.Embedding(config.max_seq_len, config.d_model)
self.embed_dropout = nn.Dropout(config.dropout)
self.embed_scale = math.sqrt(config.d_model)
# Decoder blocks
self.blocks = nn.ModuleList([
GrassmannDecoderBlock(config, i)
for i in range(config.n_decoder_layers)
])
# Output projection to vocab
self.output_norm = nn.LayerNorm(config.d_model)
self.output_proj = nn.Linear(config.d_model, config.vocab_size)
def forward(
self,
tokens: torch.Tensor,
encoder_out: torch.Tensor,
) -> torch.Tensor:
"""
Args:
tokens: (batch, T_dec) - input token indices (shifted right)
encoder_out: (batch, T_enc, d_model) - encoder hidden states
Returns:
logits: (batch, T_dec, vocab_size)
"""
B, T = tokens.shape
# Token + positional embeddings
positions = torch.arange(T, device=tokens.device).unsqueeze(0)
h = self.token_embed(tokens) * self.embed_scale + self.pos_embed(positions)
h = self.embed_dropout(h)
# Decoder blocks
for block in self.blocks:
h = block(h, encoder_out)
# Output
h = self.output_norm(h)
logits = self.output_proj(h)
return logits