duplex-0 / duplex /kernels.py
multimodalart's picture
multimodalart HF Staff
Upload folder using huggingface_hub
d20eb3e verified
Raw History Blame Contribute Delete
6.37 kB
"""Depth attention over the codebooks of one frame, specialised for short rows.
General attention kernels spend most of their tiles on padding when the sequence
is 8 or 16 tokens. On CUDA under reduced precision this packs the frames of one
head into a 16-row tile (two frames of one stream, or one frame of two streams)
with a block-diagonal causal mask, the approach of Caramel
(github.com/naklecha/caramel, kernels.py). Everywhere else it is the PyTorch
reference, which defines the math the kernel must match.
"""
import math
import torch
from torch.nn import functional as F
try:
import triton
import triton.language as tl
except ImportError:
triton = None
ROWS_PER_TILE = 16
def reference(q, k, v):
"""Causal attention for [rows, positions, heads, head_width] inputs."""
q, k, v = (y.transpose(1, 2) for y in (q, k, v))
return F.scaled_dot_product_attention(q, k, v, is_causal=True).transpose(1, 2)
if triton is not None:
@triton.jit
def _probabilities(
Q, K, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr
):
rows: tl.constexpr = FPT * POS
pid = tl.program_id(0).to(tl.int64)
r, d = tl.arange(0, rows), tl.arange(0, HD)
n, position = (pid // H) * FPT + r // POS, r % POS
valid = (n < N)[:, None]
offsets = ((n * POS + position) * H + pid % H)[:, None] * HD + d[None, :]
q = tl.load(Q + offsets, mask=valid, other=0.0)
k = tl.load(K + offsets, mask=valid, other=0.0)
s = tl.dot(q, tl.trans(k)) * scale
same_frame = (r[:, None] // POS) == (r[None, :] // POS)
causal = position[:, None] >= position[None, :]
s = tl.where(same_frame & causal, s, float("-inf"))
p = tl.exp(s - tl.max(s, axis=1)[:, None])
return q, k, p / tl.sum(p, axis=1)[:, None], offsets, valid
@triton.jit
def _forward(
Q, K, V, O, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr
):
_, _, p, offsets, valid = _probabilities(Q, K, N, H, scale, HD, POS, FPT)
v = tl.load(V + offsets, mask=valid, other=0.0)
o = tl.dot(p, v.to(tl.float32), input_precision="ieee")
tl.store(O + offsets, o.to(O.dtype.element_ty), mask=valid)
@triton.jit
def _backward(
Q,
K,
V,
DO,
DQ,
DK,
DV,
N,
H,
scale,
HD: tl.constexpr,
POS: tl.constexpr,
FPT: tl.constexpr,
):
# Recomputes the small probability tile, so forward saves only its inputs.
q, k, p, offsets, valid = _probabilities(Q, K, N, H, scale, HD, POS, FPT)
v = tl.load(V + offsets, mask=valid, other=0.0)
do = tl.load(DO + offsets, mask=valid, other=0.0)
dv = tl.dot(tl.trans(p), do.to(tl.float32), input_precision="ieee")
dp = tl.dot(do, tl.trans(v))
ds = p * (dp - tl.sum(p * dp, axis=1)[:, None]) * scale
dq = tl.dot(ds, k.to(tl.float32), input_precision="ieee")
dk = tl.dot(tl.trans(ds), q.to(tl.float32), input_precision="ieee")
ty = DQ.dtype.element_ty
tl.store(DQ + offsets, dq.to(ty), mask=valid)
tl.store(DK + offsets, dk.to(ty), mask=valid)
tl.store(DV + offsets, dv.to(ty), mask=valid)
def _launch(kernel, q, *tensors):
n, positions, heads, head_width = q.shape
frames = ROWS_PER_TILE // positions
kernel[(triton.cdiv(n, frames) * heads,)](
q,
*tensors,
n,
heads,
1.0 / math.sqrt(head_width),
HD=head_width,
POS=positions,
FPT=frames,
num_warps=1,
)
# Custom ops make each kernel one opaque node for torch.compile.
@torch.library.custom_op("duplex::depth_attention", mutates_args=())
def _attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
q, k, v = q.contiguous(), k.contiguous(), v.contiguous()
o = torch.empty_like(q)
_launch(_forward, q, k, v, o)
return o
@torch.library.custom_op("duplex::depth_attention_backward", mutates_args=())
def _attention_backward(
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, do: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
q, k, v, do = q.contiguous(), k.contiguous(), v.contiguous(), do.contiguous()
dq, dk, dv = torch.empty_like(q), torch.empty_like(q), torch.empty_like(q)
_launch(_backward, q, k, v, do, dq, dk, dv)
return dq, dk, dv
_attention.register_fake(lambda q, k, v: torch.empty_like(q))
_attention_backward.register_fake(
lambda q, k, v, do: (
torch.empty_like(q),
torch.empty_like(q),
torch.empty_like(q),
)
)
_attention.register_autograd(
lambda ctx, grad: _attention_backward(*ctx.saved_tensors, grad),
setup_context=lambda ctx, inputs, output: ctx.save_for_backward(*inputs),
)
def depth_attention(q, k, v):
"""q, k and v are [rows, positions, heads, head_width]; so is the result."""
if (
triton is None
or not q.is_cuda
or q.dtype == torch.float32
or ROWS_PER_TILE % q.shape[1]
):
return reference(q, k, v)
return _attention(q, k, v)
if __name__ == "__main__":
# GPU check: the packed kernel against the reference for one and two streams.
for positions in (8, 16):
q, k, v = (
torch.randn(
4096,
positions,
4,
64,
device="cuda",
dtype=torch.bfloat16,
requires_grad=True,
)
for _ in range(3)
)
grad = torch.randn_like(q)
results = []
for function in (depth_attention, reference):
out = function(q, k, v)
out.backward(grad)
results.append([out.float(), *(x.grad.float() for x in (q, k, v))])
for x in (q, k, v):
x.grad = None
errors = [
((a - b).norm() / b.norm()).item() for a, b in zip(*results, strict=True)
]
print({"positions": positions, "relative_errors": errors}, flush=True)
if max(errors) > 0.01:
raise SystemExit("kernel differs from reference")