Download ar/model/attention.py from AjayRangoji/AR: direct link, hf CLI and curl.
- Browser
- Download file 2.3 kB
-
https://huggingface.co/AjayRangoji/AR/resolve/main/ar/model/attention.py
- Command line
-
hf download hf://AjayRangoji/AR/ar/model/attention.py
-
curl -L -o attention.py https://huggingface.co/AjayRangoji/AR/resolve/main/ar/model/attention.py
2.3 kB
| import torch | |
| import torch.nn as nn | |
| from torch.nn.attention.flex_attention import flex_attention | |
| from .rmsnorm import RMSNorm | |
| from .rope import apply_rope | |
| # compile FlexAttention once at import; without compile it falls back | |
| # to a very slow reference kernel | |
| flex_attention = torch.compile(flex_attention) | |
| class GroupedQueryAttention(nn.Module): | |
| """Grouped-query attention with RoPE, QK-norm, and a caller-supplied mask. | |
| Query heads are split into groups that each share one key/value head, | |
| shrinking the KV cache with almost no loss in quality. The block_mask | |
| is built outside the model (causal + same-document) and simply passed in. | |
| """ | |
| def __init__(self, dim: int, n_heads: int, n_kv_heads: int, head_dim: int): | |
| super().__init__() | |
| self.n_heads = n_heads | |
| self.n_kv_heads = n_kv_heads | |
| self.head_dim = head_dim | |
| # projections: Q is full width, K/V are narrower (fewer heads) | |
| self.q_proj = nn.Linear(dim, n_heads * head_dim, bias=False) | |
| self.k_proj = nn.Linear(dim, n_kv_heads * head_dim, bias=False) | |
| self.v_proj = nn.Linear(dim, n_kv_heads * head_dim, bias=False) | |
| self.o_proj = nn.Linear(n_heads * head_dim, dim, bias=False) | |
| # QK-norm: normalize each head's Q and K before attention (stability) | |
| self.q_norm = RMSNorm(head_dim) | |
| self.k_norm = RMSNorm(head_dim) | |
| def forward(self, x, cos, sin, block_mask): | |
| B, T, _ = x.shape | |
| # project, then split into heads: (B, heads, T, head_dim) | |
| q = self.q_proj(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2) | |
| k = self.k_proj(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2) | |
| v = self.v_proj(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2) | |
| # QK-norm, then rotate Q and K by their positions | |
| q = apply_rope(self.q_norm(q), cos, sin) | |
| k = apply_rope(self.k_norm(k), cos, sin) | |
| # FlexAttention handles GQA internally; no KV expansion required, | |
| # which saves memory vs the old repeat_interleave path | |
| out = flex_attention(q, k, v, block_mask=block_mask, enable_gqa=True) | |
| # merge heads back together and project out | |
| out = out.transpose(1, 2).contiguous().view(B, T, -1) | |
| return self.o_proj(out) |