AR / ar /model /attention.py
AjayRangoji's picture
Upload AR-4B pretraining codebase, configs, and cluster scripts
2ae6691 verified
Raw History Blame Contribute Delete
2.3 kB
import torch
import torch.nn as nn
from torch.nn.attention.flex_attention import flex_attention
from .rmsnorm import RMSNorm
from .rope import apply_rope
# compile FlexAttention once at import; without compile it falls back
# to a very slow reference kernel
flex_attention = torch.compile(flex_attention)
class GroupedQueryAttention(nn.Module):
"""Grouped-query attention with RoPE, QK-norm, and a caller-supplied mask.
Query heads are split into groups that each share one key/value head,
shrinking the KV cache with almost no loss in quality. The block_mask
is built outside the model (causal + same-document) and simply passed in.
"""
def __init__(self, dim: int, n_heads: int, n_kv_heads: int, head_dim: int):
super().__init__()
self.n_heads = n_heads
self.n_kv_heads = n_kv_heads
self.head_dim = head_dim
# projections: Q is full width, K/V are narrower (fewer heads)
self.q_proj = nn.Linear(dim, n_heads * head_dim, bias=False)
self.k_proj = nn.Linear(dim, n_kv_heads * head_dim, bias=False)
self.v_proj = nn.Linear(dim, n_kv_heads * head_dim, bias=False)
self.o_proj = nn.Linear(n_heads * head_dim, dim, bias=False)
# QK-norm: normalize each head's Q and K before attention (stability)
self.q_norm = RMSNorm(head_dim)
self.k_norm = RMSNorm(head_dim)
def forward(self, x, cos, sin, block_mask):
B, T, _ = x.shape
# project, then split into heads: (B, heads, T, head_dim)
q = self.q_proj(x).view(B, T, self.n_heads, self.head_dim).transpose(1, 2)
k = self.k_proj(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
v = self.v_proj(x).view(B, T, self.n_kv_heads, self.head_dim).transpose(1, 2)
# QK-norm, then rotate Q and K by their positions
q = apply_rope(self.q_norm(q), cos, sin)
k = apply_rope(self.k_norm(k), cos, sin)
# FlexAttention handles GQA internally; no KV expansion required,
# which saves memory vs the old repeat_interleave path
out = flex_attention(q, k, v, block_mask=block_mask, enable_gqa=True)
# merge heads back together and project out
out = out.transpose(1, 2).contiguous().view(B, T, -1)
return self.o_proj(out)