TheFireHacker's picture
Upload folder using huggingface_hub
27cd743 verified
Raw History Blame Contribute Delete
2.98 kB
import torch
import torch.nn as nn
import torch.nn.functional as F
from .norm import ZeroCenteredRMSNorm
from .rope import apply_rope
class Attention(nn.Module):
"""GQA attention with per-head QK-norm (applied before RoPE), RoPE, and an
elementwise output gate: Y = sdpa_out * sigmoid(X @ W_g), applied before o_proj.
"""
def __init__(
self,
d_model: int,
n_heads: int,
head_dim: int,
n_kv_heads: int,
norm_eps: float,
output_gate: bool,
bias: bool,
):
super().__init__()
self.n_heads = n_heads
self.head_dim = head_dim
self.n_kv_heads = n_kv_heads
self.output_gate = output_gate
self.q_proj = nn.Linear(d_model, n_heads * head_dim, bias=bias)
self.k_proj = nn.Linear(d_model, n_kv_heads * head_dim, bias=bias)
self.v_proj = nn.Linear(d_model, n_kv_heads * head_dim, bias=bias)
self.o_proj = nn.Linear(n_heads * head_dim, d_model, bias=bias)
self.q_norm = ZeroCenteredRMSNorm(head_dim, eps=norm_eps)
self.k_norm = ZeroCenteredRMSNorm(head_dim, eps=norm_eps)
if output_gate:
self.gate_proj = nn.Linear(d_model, n_heads * head_dim, bias=bias)
def forward(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> torch.Tensor:
bsz, seq_len, _ = x.shape
q = self.q_proj(x).view(bsz, seq_len, self.n_heads, self.head_dim)
k = self.k_proj(x).view(bsz, seq_len, self.n_kv_heads, self.head_dim)
v = self.v_proj(x).view(bsz, seq_len, self.n_kv_heads, self.head_dim)
q = self.q_norm(q)
k = self.k_norm(k)
q = apply_rope(q.transpose(1, 2), cos, sin) # (bsz, n_heads, seq_len, head_dim)
k = apply_rope(k.transpose(1, 2), cos, sin)
v = v.transpose(1, 2)
attn_out = F.scaled_dot_product_attention(
q, k, v, is_causal=True, enable_gqa=(self.n_heads != self.n_kv_heads)
)
attn_out = attn_out.transpose(1, 2).contiguous().view(bsz, seq_len, self.n_heads * self.head_dim)
if self.output_gate:
gate = torch.sigmoid(self.gate_proj(x))
attn_out = attn_out * gate
return self.o_proj(attn_out)
@torch.no_grad()
def max_attn_logit(self, x: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor) -> float:
"""Unfused recompute of pre-softmax attention logits, for periodic diagnostics only."""
bsz, seq_len, _ = x.shape
q = self.q_norm(self.q_proj(x).view(bsz, seq_len, self.n_heads, self.head_dim))
k = self.k_norm(self.k_proj(x).view(bsz, seq_len, self.n_kv_heads, self.head_dim))
q = apply_rope(q.transpose(1, 2), cos, sin)
k = apply_rope(k.transpose(1, 2), cos, sin)
rep = self.n_heads // self.n_kv_heads
k = k.repeat_interleave(rep, dim=1)
logits = (q.float() @ k.float().transpose(-1, -2)) / (self.head_dim ** 0.5)
return logits.abs().max().item()