Spaces:
Running
Running
Nicholas Finch
fix: default workers=0 (sequential) for Docker cache — ProcessPoolExecutor hangs on cpu-basic
d7013e6 Download src/models/grassmann_decoder.py from tisaconundrum/grassmann-cache: direct link, hf CLI and curl.
- Browser
- Download file 3.99 kB
-
https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/src/models/grassmann_decoder.py
- Command line
-
hf download hf://spaces/tisaconundrum/grassmann-cache/src/models/grassmann_decoder.py
-
curl -L -o grassmann_decoder.py https://huggingface.co/spaces/tisaconundrum/grassmann-cache/resolve/main/src/models/grassmann_decoder.py
3.99 kB
| """ | |
| Grassmann Decoder. | |
| A fully attention-free autoregressive decoder using: | |
| 1. Causal Grassmann self-mixing (same as encoder) | |
| 2. Grassmann cross-mixing (replaces cross-attention) | |
| 3. Position-wise FFN | |
| Each decoder block: CausalSelfMix → CrossMix → FFN | |
| Stacked M times, with token embeddings and output projection. | |
| """ | |
| import math | |
| import torch | |
| import torch.nn as nn | |
| from .config import GrassmannConfig | |
| from .grassmann_layer import CausalGrassmannLayer | |
| from .grassmann_cross_layer import GrassmannCrossLayer | |
| class GrassmannDecoderBlock(nn.Module): | |
| """ | |
| One decoder block: self-mixing → cross-mixing → FFN. | |
| The self-mixing uses CausalGrassmannLayer (causal, decoder-to-decoder). | |
| The cross-mixing uses GrassmannCrossLayer (decoder-to-encoder). | |
| The FFN is already included in CausalGrassmannLayer, so we only add | |
| the cross-mixing step between self-mixing and the FFN residual. | |
| """ | |
| def __init__(self, config: GrassmannConfig, layer_idx: int = 0): | |
| super().__init__() | |
| d = config.d_model | |
| # Causal self-mixing (includes FFN) | |
| self.self_mix = CausalGrassmannLayer(config, layer_idx) | |
| # Grassmann cross-mixing with encoder | |
| self.cross_mix = GrassmannCrossLayer(config) | |
| # Additional FFN after cross-mixing | |
| self.ffn = nn.Sequential( | |
| nn.Linear(d, config.d_ff), | |
| nn.GELU(), | |
| nn.Dropout(config.dropout), | |
| nn.Linear(config.d_ff, d), | |
| nn.Dropout(config.dropout), | |
| ) | |
| self.norm_ffn = nn.LayerNorm(d) | |
| def forward(self, h_dec: torch.Tensor, h_enc: torch.Tensor) -> torch.Tensor: | |
| """ | |
| Args: | |
| h_dec: (batch, T_dec, d_model) - decoder states | |
| h_enc: (batch, T_enc, d_model) - encoder output | |
| Returns: | |
| (batch, T_dec, d_model) - updated decoder states | |
| """ | |
| # Self-mixing (causal, within decoder sequence) | |
| h = self.self_mix(h_dec) | |
| # Cross-mixing (decoder ↔ encoder via Grassmann geometry) | |
| h = self.cross_mix(h, h_enc) | |
| # FFN with residual | |
| h_out = self.norm_ffn(h + self.ffn(h)) | |
| return h_out | |
| class GrassmannDecoder(nn.Module): | |
| """ | |
| Full Grassmann decoder: token embeddings → M decoder blocks → output logits. | |
| Fully attention-free. All sequence interactions happen through | |
| Plücker coordinates on Gr(2, r). | |
| """ | |
| def __init__(self, config: GrassmannConfig): | |
| super().__init__() | |
| self.config = config | |
| # Token embeddings | |
| self.token_embed = nn.Embedding(config.vocab_size, config.d_model) | |
| self.pos_embed = nn.Embedding(config.max_seq_len, config.d_model) | |
| self.embed_dropout = nn.Dropout(config.dropout) | |
| self.embed_scale = math.sqrt(config.d_model) | |
| # Decoder blocks | |
| self.blocks = nn.ModuleList([ | |
| GrassmannDecoderBlock(config, i) | |
| for i in range(config.n_decoder_layers) | |
| ]) | |
| # Output projection to vocab | |
| self.output_norm = nn.LayerNorm(config.d_model) | |
| self.output_proj = nn.Linear(config.d_model, config.vocab_size) | |
| def forward( | |
| self, | |
| tokens: torch.Tensor, | |
| encoder_out: torch.Tensor, | |
| ) -> torch.Tensor: | |
| """ | |
| Args: | |
| tokens: (batch, T_dec) - input token indices (shifted right) | |
| encoder_out: (batch, T_enc, d_model) - encoder hidden states | |
| Returns: | |
| logits: (batch, T_dec, vocab_size) | |
| """ | |
| B, T = tokens.shape | |
| # Token + positional embeddings | |
| positions = torch.arange(T, device=tokens.device).unsqueeze(0) | |
| h = self.token_embed(tokens) * self.embed_scale + self.pos_embed(positions) | |
| h = self.embed_dropout(h) | |
| # Decoder blocks | |
| for block in self.blocks: | |
| h = block(h, encoder_out) | |
| # Output | |
| h = self.output_norm(h) | |
| logits = self.output_proj(h) | |
| return logits | |