"""Depth attention over the codebooks of one frame, specialised for short rows. General attention kernels spend most of their tiles on padding when the sequence is 8 or 16 tokens. On CUDA under reduced precision this packs the frames of one head into a 16-row tile (two frames of one stream, or one frame of two streams) with a block-diagonal causal mask, the approach of Caramel (github.com/naklecha/caramel, kernels.py). Everywhere else it is the PyTorch reference, which defines the math the kernel must match. """ import math import torch from torch.nn import functional as F try: import triton import triton.language as tl except ImportError: triton = None ROWS_PER_TILE = 16 def reference(q, k, v): """Causal attention for [rows, positions, heads, head_width] inputs.""" q, k, v = (y.transpose(1, 2) for y in (q, k, v)) return F.scaled_dot_product_attention(q, k, v, is_causal=True).transpose(1, 2) if triton is not None: @triton.jit def _probabilities( Q, K, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr ): rows: tl.constexpr = FPT * POS pid = tl.program_id(0).to(tl.int64) r, d = tl.arange(0, rows), tl.arange(0, HD) n, position = (pid // H) * FPT + r // POS, r % POS valid = (n < N)[:, None] offsets = ((n * POS + position) * H + pid % H)[:, None] * HD + d[None, :] q = tl.load(Q + offsets, mask=valid, other=0.0) k = tl.load(K + offsets, mask=valid, other=0.0) s = tl.dot(q, tl.trans(k)) * scale same_frame = (r[:, None] // POS) == (r[None, :] // POS) causal = position[:, None] >= position[None, :] s = tl.where(same_frame & causal, s, float("-inf")) p = tl.exp(s - tl.max(s, axis=1)[:, None]) return q, k, p / tl.sum(p, axis=1)[:, None], offsets, valid @triton.jit def _forward( Q, K, V, O, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr ): _, _, p, offsets, valid = _probabilities(Q, K, N, H, scale, HD, POS, FPT) v = tl.load(V + offsets, mask=valid, other=0.0) o = tl.dot(p, v.to(tl.float32), input_precision="ieee") tl.store(O + offsets, o.to(O.dtype.element_ty), mask=valid) @triton.jit def _backward( Q, K, V, DO, DQ, DK, DV, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr, ): # Recomputes the small probability tile, so forward saves only its inputs. q, k, p, offsets, valid = _probabilities(Q, K, N, H, scale, HD, POS, FPT) v = tl.load(V + offsets, mask=valid, other=0.0) do = tl.load(DO + offsets, mask=valid, other=0.0) dv = tl.dot(tl.trans(p), do.to(tl.float32), input_precision="ieee") dp = tl.dot(do, tl.trans(v)) ds = p * (dp - tl.sum(p * dp, axis=1)[:, None]) * scale dq = tl.dot(ds, k.to(tl.float32), input_precision="ieee") dk = tl.dot(tl.trans(ds), q.to(tl.float32), input_precision="ieee") ty = DQ.dtype.element_ty tl.store(DQ + offsets, dq.to(ty), mask=valid) tl.store(DK + offsets, dk.to(ty), mask=valid) tl.store(DV + offsets, dv.to(ty), mask=valid) def _launch(kernel, q, *tensors): n, positions, heads, head_width = q.shape frames = ROWS_PER_TILE // positions kernel[(triton.cdiv(n, frames) * heads,)]( q, *tensors, n, heads, 1.0 / math.sqrt(head_width), HD=head_width, POS=positions, FPT=frames, num_warps=1, ) # Custom ops make each kernel one opaque node for torch.compile. @torch.library.custom_op("duplex::depth_attention", mutates_args=()) def _attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor: q, k, v = q.contiguous(), k.contiguous(), v.contiguous() o = torch.empty_like(q) _launch(_forward, q, k, v, o) return o @torch.library.custom_op("duplex::depth_attention_backward", mutates_args=()) def _attention_backward( q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, do: torch.Tensor ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: q, k, v, do = q.contiguous(), k.contiguous(), v.contiguous(), do.contiguous() dq, dk, dv = torch.empty_like(q), torch.empty_like(q), torch.empty_like(q) _launch(_backward, q, k, v, do, dq, dk, dv) return dq, dk, dv _attention.register_fake(lambda q, k, v: torch.empty_like(q)) _attention_backward.register_fake( lambda q, k, v, do: ( torch.empty_like(q), torch.empty_like(q), torch.empty_like(q), ) ) _attention.register_autograd( lambda ctx, grad: _attention_backward(*ctx.saved_tensors, grad), setup_context=lambda ctx, inputs, output: ctx.save_for_backward(*inputs), ) def depth_attention(q, k, v): """q, k and v are [rows, positions, heads, head_width]; so is the result.""" if ( triton is None or not q.is_cuda or q.dtype == torch.float32 or ROWS_PER_TILE % q.shape[1] ): return reference(q, k, v) return _attention(q, k, v) if __name__ == "__main__": # GPU check: the packed kernel against the reference for one and two streams. for positions in (8, 16): q, k, v = ( torch.randn( 4096, positions, 4, 64, device="cuda", dtype=torch.bfloat16, requires_grad=True, ) for _ in range(3) ) grad = torch.randn_like(q) results = [] for function in (depth_attention, reference): out = function(q, k, v) out.backward(grad) results.append([out.float(), *(x.grad.float() for x in (q, k, v))]) for x in (q, k, v): x.grad = None errors = [ ((a - b).norm() / b.norm()).item() for a, b in zip(*results, strict=True) ] print({"positions": positions, "relative_errors": errors}, flush=True) if max(errors) > 0.01: raise SystemExit("kernel differs from reference")