Spaces:
Running on Zero
Running on Zero
File size: 6,373 Bytes
d20eb3e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 | """Depth attention over the codebooks of one frame, specialised for short rows.
General attention kernels spend most of their tiles on padding when the sequence
is 8 or 16 tokens. On CUDA under reduced precision this packs the frames of one
head into a 16-row tile (two frames of one stream, or one frame of two streams)
with a block-diagonal causal mask, the approach of Caramel
(github.com/naklecha/caramel, kernels.py). Everywhere else it is the PyTorch
reference, which defines the math the kernel must match.
"""
import math
import torch
from torch.nn import functional as F
try:
import triton
import triton.language as tl
except ImportError:
triton = None
ROWS_PER_TILE = 16
def reference(q, k, v):
"""Causal attention for [rows, positions, heads, head_width] inputs."""
q, k, v = (y.transpose(1, 2) for y in (q, k, v))
return F.scaled_dot_product_attention(q, k, v, is_causal=True).transpose(1, 2)
if triton is not None:
@triton.jit
def _probabilities(
Q, K, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr
):
rows: tl.constexpr = FPT * POS
pid = tl.program_id(0).to(tl.int64)
r, d = tl.arange(0, rows), tl.arange(0, HD)
n, position = (pid // H) * FPT + r // POS, r % POS
valid = (n < N)[:, None]
offsets = ((n * POS + position) * H + pid % H)[:, None] * HD + d[None, :]
q = tl.load(Q + offsets, mask=valid, other=0.0)
k = tl.load(K + offsets, mask=valid, other=0.0)
s = tl.dot(q, tl.trans(k)) * scale
same_frame = (r[:, None] // POS) == (r[None, :] // POS)
causal = position[:, None] >= position[None, :]
s = tl.where(same_frame & causal, s, float("-inf"))
p = tl.exp(s - tl.max(s, axis=1)[:, None])
return q, k, p / tl.sum(p, axis=1)[:, None], offsets, valid
@triton.jit
def _forward(
Q, K, V, O, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr
):
_, _, p, offsets, valid = _probabilities(Q, K, N, H, scale, HD, POS, FPT)
v = tl.load(V + offsets, mask=valid, other=0.0)
o = tl.dot(p, v.to(tl.float32), input_precision="ieee")
tl.store(O + offsets, o.to(O.dtype.element_ty), mask=valid)
@triton.jit
def _backward(
Q,
K,
V,
DO,
DQ,
DK,
DV,
N,
H,
scale,
HD: tl.constexpr,
POS: tl.constexpr,
FPT: tl.constexpr,
):
# Recomputes the small probability tile, so forward saves only its inputs.
q, k, p, offsets, valid = _probabilities(Q, K, N, H, scale, HD, POS, FPT)
v = tl.load(V + offsets, mask=valid, other=0.0)
do = tl.load(DO + offsets, mask=valid, other=0.0)
dv = tl.dot(tl.trans(p), do.to(tl.float32), input_precision="ieee")
dp = tl.dot(do, tl.trans(v))
ds = p * (dp - tl.sum(p * dp, axis=1)[:, None]) * scale
dq = tl.dot(ds, k.to(tl.float32), input_precision="ieee")
dk = tl.dot(tl.trans(ds), q.to(tl.float32), input_precision="ieee")
ty = DQ.dtype.element_ty
tl.store(DQ + offsets, dq.to(ty), mask=valid)
tl.store(DK + offsets, dk.to(ty), mask=valid)
tl.store(DV + offsets, dv.to(ty), mask=valid)
def _launch(kernel, q, *tensors):
n, positions, heads, head_width = q.shape
frames = ROWS_PER_TILE // positions
kernel[(triton.cdiv(n, frames) * heads,)](
q,
*tensors,
n,
heads,
1.0 / math.sqrt(head_width),
HD=head_width,
POS=positions,
FPT=frames,
num_warps=1,
)
# Custom ops make each kernel one opaque node for torch.compile.
@torch.library.custom_op("duplex::depth_attention", mutates_args=())
def _attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor:
q, k, v = q.contiguous(), k.contiguous(), v.contiguous()
o = torch.empty_like(q)
_launch(_forward, q, k, v, o)
return o
@torch.library.custom_op("duplex::depth_attention_backward", mutates_args=())
def _attention_backward(
q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, do: torch.Tensor
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
q, k, v, do = q.contiguous(), k.contiguous(), v.contiguous(), do.contiguous()
dq, dk, dv = torch.empty_like(q), torch.empty_like(q), torch.empty_like(q)
_launch(_backward, q, k, v, do, dq, dk, dv)
return dq, dk, dv
_attention.register_fake(lambda q, k, v: torch.empty_like(q))
_attention_backward.register_fake(
lambda q, k, v, do: (
torch.empty_like(q),
torch.empty_like(q),
torch.empty_like(q),
)
)
_attention.register_autograd(
lambda ctx, grad: _attention_backward(*ctx.saved_tensors, grad),
setup_context=lambda ctx, inputs, output: ctx.save_for_backward(*inputs),
)
def depth_attention(q, k, v):
"""q, k and v are [rows, positions, heads, head_width]; so is the result."""
if (
triton is None
or not q.is_cuda
or q.dtype == torch.float32
or ROWS_PER_TILE % q.shape[1]
):
return reference(q, k, v)
return _attention(q, k, v)
if __name__ == "__main__":
# GPU check: the packed kernel against the reference for one and two streams.
for positions in (8, 16):
q, k, v = (
torch.randn(
4096,
positions,
4,
64,
device="cuda",
dtype=torch.bfloat16,
requires_grad=True,
)
for _ in range(3)
)
grad = torch.randn_like(q)
results = []
for function in (depth_attention, reference):
out = function(q, k, v)
out.backward(grad)
results.append([out.float(), *(x.grad.float() for x in (q, k, v))])
for x in (q, k, v):
x.grad = None
errors = [
((a - b).norm() / b.norm()).item() for a, b in zip(*results, strict=True)
]
print({"positions": positions, "relative_errors": errors}, flush=True)
if max(errors) > 0.01:
raise SystemExit("kernel differs from reference")
|