Download src/model/attention.py from TheFireHacker/bubblspace-moe0.9b-a0.2b: direct link, hf CLI and curl.
- Browser
- Download file 2.98 kB
-
https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/model/attention.py
- Command line
-
hf download hf://TheFireHacker/bubblspace-moe0.9b-a0.2b/src/model/attention.py
-
curl -L -o attention.py https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/src/model/attention.py
2.98 kB
| import torch | |
| import torch.nn as nn | |
| import torch.nn.functional as F | |
| from .norm import ZeroCenteredRMSNorm | |
| from .rope import apply_rope | |
| class Attention(nn.Module): | |
| """GQA attention with per-head QK-norm (applied before RoPE), RoPE, and an | |
| elementwise output gate: Y = sdpa_out * sigmoid(X @ W_g), applied before o_proj. | |
| """ | |
| def __init__( | |
| self, | |
| d_model: int, | |
| n_heads: int, | |
| head_dim: int, | |
| n_kv_heads: int, | |
| norm_eps: float, | |
| output_gate: bool, | |
| bias: bool, | |
| ): | |
| super().__init__() | |
| self.n_heads = n_heads | |
| self.head_dim = head_dim | |
| self.n_kv_heads = n_kv_heads | |
| self.output_gate = output_gate | |
| self.q_proj = nn.Linear(d_model, n_heads * head_dim, bias=bias) | |
| self.k_proj = nn.Linear(d_model, n_kv_heads * head_dim, bias=bias) | |
| self.v_proj = nn.Linear(d_model, n_kv_heads * head_dim, bias=bias) | |
| self.o_proj = nn.Linear(n_heads * head_dim, d_model, bias=bias) | |
| self.q_norm = ZeroCenteredRMSNorm(head_dim, eps=norm_eps) | |
| self.k_norm = ZeroCenteredRMSNorm(head_dim, eps=norm_eps) | |
| if output_gate: | |
| self.gate_proj = nn.Linear(d_model, n_heads * head_dim, bias=bias) | |
| def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor: | |
| bsz, seq_len, _ = x.shape | |
| q = self.q_proj(x).view(bsz, seq_len, self.n_heads, self.head_dim) | |
| k = self.k_proj(x).view(bsz, seq_len, self.n_kv_heads, self.head_dim) | |
| v = self.v_proj(x).view(bsz, seq_len, self.n_kv_heads, self.head_dim) | |
| q = self.q_norm(q) | |
| k = self.k_norm(k) | |
| q = apply_rope(q.transpose(1, 2), cos, sin) # (bsz, n_heads, seq_len, head_dim) | |
| k = apply_rope(k.transpose(1, 2), cos, sin) | |
| v = v.transpose(1, 2) | |
| attn_out = F.scaled_dot_product_attention( | |
| q, k, v, is_causal=True, enable_gqa=(self.n_heads != self.n_kv_heads) | |
| ) | |
| attn_out = attn_out.transpose(1, 2).contiguous().view(bsz, seq_len, self.n_heads * self.head_dim) | |
| if self.output_gate: | |
| gate = torch.sigmoid(self.gate_proj(x)) | |
| attn_out = attn_out * gate | |
| return self.o_proj(attn_out) | |
| def max_attn_logit(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> float: | |
| """Unfused recompute of pre-softmax attention logits, for periodic diagnostics only.""" | |
| bsz, seq_len, _ = x.shape | |
| q = self.q_norm(self.q_proj(x).view(bsz, seq_len, self.n_heads, self.head_dim)) | |
| k = self.k_norm(self.k_proj(x).view(bsz, seq_len, self.n_kv_heads, self.head_dim)) | |
| q = apply_rope(q.transpose(1, 2), cos, sin) | |
| k = apply_rope(k.transpose(1, 2), cos, sin) | |
| rep = self.n_heads // self.n_kv_heads | |
| k = k.repeat_interleave(rep, dim=1) | |
| logits = (q.float() @ k.float().transpose(-1, -2)) / (self.head_dim ** 0.5) | |
| return logits.abs().max().item() | |