Spaces:
Runtime error
Runtime error
File size: 868 Bytes
cc15101 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 | """
One transformer block: a pre-norm residual around attention, then another
around the SwiGLU MLP
x = x + attn(norm1(x))
x = x + mlp(norm2(x))
Pre-norm (norm before each sublayer) keeps a clean residual highway, which
lets deep stacks train stably.
"""
import torch.nn as nn
from .norm import AbyssRMSNorm
from .attention import AbyssCausalSelfAttention
from .mlp import AbyssSwiGLU
class AbyssBlock(nn.Module):
def __init__(self, config):
super().__init__()
self.norm1 = AbyssRMSNorm(config.n_embd, config.norm_eps)
self.attn = AbyssCausalSelfAttention(config)
self.norm2 = AbyssRMSNorm(config.n_embd, config.norm_eps)
self.mlp = AbyssSwiGLU(config)
def forward(self, x):
x = x + self.attn(self.norm1(x))
x = x + self.mlp(self.norm2(x))
return x |