Download scripts/duplex/model/kernels.py from Trelis/duplex-0: direct link, hf CLI and curl.
- Browser
- Download file 6.37 kB
-
https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/model/kernels.py
- Command line
-
hf download hf://Trelis/duplex-0/scripts/duplex/model/kernels.py
-
curl -L -o kernels.py https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/model/kernels.py
6.37 kB
| """Depth attention over the codebooks of one frame, specialised for short rows. | |
| General attention kernels spend most of their tiles on padding when the sequence | |
| is 8 or 16 tokens. On CUDA under reduced precision this packs the frames of one | |
| head into a 16-row tile (two frames of one stream, or one frame of two streams) | |
| with a block-diagonal causal mask, the approach of Caramel | |
| (github.com/naklecha/caramel, kernels.py). Everywhere else it is the PyTorch | |
| reference, which defines the math the kernel must match. | |
| """ | |
| import math | |
| import torch | |
| from torch.nn import functional as F | |
| try: | |
| import triton | |
| import triton.language as tl | |
| except ImportError: | |
| triton = None | |
| ROWS_PER_TILE = 16 | |
| def reference(q, k, v): | |
| """Causal attention for [rows, positions, heads, head_width] inputs.""" | |
| q, k, v = (y.transpose(1, 2) for y in (q, k, v)) | |
| return F.scaled_dot_product_attention(q, k, v, is_causal=True).transpose(1, 2) | |
| if triton is not None: | |
| def _probabilities( | |
| Q, K, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr | |
| ): | |
| rows: tl.constexpr = FPT * POS | |
| pid = tl.program_id(0).to(tl.int64) | |
| r, d = tl.arange(0, rows), tl.arange(0, HD) | |
| n, position = (pid // H) * FPT + r // POS, r % POS | |
| valid = (n < N)[:, None] | |
| offsets = ((n * POS + position) * H + pid % H)[:, None] * HD + d[None, :] | |
| q = tl.load(Q + offsets, mask=valid, other=0.0) | |
| k = tl.load(K + offsets, mask=valid, other=0.0) | |
| s = tl.dot(q, tl.trans(k)) * scale | |
| same_frame = (r[:, None] // POS) == (r[None, :] // POS) | |
| causal = position[:, None] >= position[None, :] | |
| s = tl.where(same_frame & causal, s, float("-inf")) | |
| p = tl.exp(s - tl.max(s, axis=1)[:, None]) | |
| return q, k, p / tl.sum(p, axis=1)[:, None], offsets, valid | |
| def _forward( | |
| Q, K, V, O, N, H, scale, HD: tl.constexpr, POS: tl.constexpr, FPT: tl.constexpr | |
| ): | |
| _, _, p, offsets, valid = _probabilities(Q, K, N, H, scale, HD, POS, FPT) | |
| v = tl.load(V + offsets, mask=valid, other=0.0) | |
| o = tl.dot(p, v.to(tl.float32), input_precision="ieee") | |
| tl.store(O + offsets, o.to(O.dtype.element_ty), mask=valid) | |
| def _backward( | |
| Q, | |
| K, | |
| V, | |
| DO, | |
| DQ, | |
| DK, | |
| DV, | |
| N, | |
| H, | |
| scale, | |
| HD: tl.constexpr, | |
| POS: tl.constexpr, | |
| FPT: tl.constexpr, | |
| ): | |
| # Recomputes the small probability tile, so forward saves only its inputs. | |
| q, k, p, offsets, valid = _probabilities(Q, K, N, H, scale, HD, POS, FPT) | |
| v = tl.load(V + offsets, mask=valid, other=0.0) | |
| do = tl.load(DO + offsets, mask=valid, other=0.0) | |
| dv = tl.dot(tl.trans(p), do.to(tl.float32), input_precision="ieee") | |
| dp = tl.dot(do, tl.trans(v)) | |
| ds = p * (dp - tl.sum(p * dp, axis=1)[:, None]) * scale | |
| dq = tl.dot(ds, k.to(tl.float32), input_precision="ieee") | |
| dk = tl.dot(tl.trans(ds), q.to(tl.float32), input_precision="ieee") | |
| ty = DQ.dtype.element_ty | |
| tl.store(DQ + offsets, dq.to(ty), mask=valid) | |
| tl.store(DK + offsets, dk.to(ty), mask=valid) | |
| tl.store(DV + offsets, dv.to(ty), mask=valid) | |
| def _launch(kernel, q, *tensors): | |
| n, positions, heads, head_width = q.shape | |
| frames = ROWS_PER_TILE // positions | |
| kernel[(triton.cdiv(n, frames) * heads,)]( | |
| q, | |
| *tensors, | |
| n, | |
| heads, | |
| 1.0 / math.sqrt(head_width), | |
| HD=head_width, | |
| POS=positions, | |
| FPT=frames, | |
| num_warps=1, | |
| ) | |
| # Custom ops make each kernel one opaque node for torch.compile. | |
| def _attention(q: torch.Tensor, k: torch.Tensor, v: torch.Tensor) -> torch.Tensor: | |
| q, k, v = q.contiguous(), k.contiguous(), v.contiguous() | |
| o = torch.empty_like(q) | |
| _launch(_forward, q, k, v, o) | |
| return o | |
| def _attention_backward( | |
| q: torch.Tensor, k: torch.Tensor, v: torch.Tensor, do: torch.Tensor | |
| ) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: | |
| q, k, v, do = q.contiguous(), k.contiguous(), v.contiguous(), do.contiguous() | |
| dq, dk, dv = torch.empty_like(q), torch.empty_like(q), torch.empty_like(q) | |
| _launch(_backward, q, k, v, do, dq, dk, dv) | |
| return dq, dk, dv | |
| _attention.register_fake(lambda q, k, v: torch.empty_like(q)) | |
| _attention_backward.register_fake( | |
| lambda q, k, v, do: ( | |
| torch.empty_like(q), | |
| torch.empty_like(q), | |
| torch.empty_like(q), | |
| ) | |
| ) | |
| _attention.register_autograd( | |
| lambda ctx, grad: _attention_backward(*ctx.saved_tensors, grad), | |
| setup_context=lambda ctx, inputs, output: ctx.save_for_backward(*inputs), | |
| ) | |
| def depth_attention(q, k, v): | |
| """q, k and v are [rows, positions, heads, head_width]; so is the result.""" | |
| if ( | |
| triton is None | |
| or not q.is_cuda | |
| or q.dtype == torch.float32 | |
| or ROWS_PER_TILE % q.shape[1] | |
| ): | |
| return reference(q, k, v) | |
| return _attention(q, k, v) | |
| if __name__ == "__main__": | |
| # GPU check: the packed kernel against the reference for one and two streams. | |
| for positions in (8, 16): | |
| q, k, v = ( | |
| torch.randn( | |
| 4096, | |
| positions, | |
| 4, | |
| 64, | |
| device="cuda", | |
| dtype=torch.bfloat16, | |
| requires_grad=True, | |
| ) | |
| for _ in range(3) | |
| ) | |
| grad = torch.randn_like(q) | |
| results = [] | |
| for function in (depth_attention, reference): | |
| out = function(q, k, v) | |
| out.backward(grad) | |
| results.append([out.float(), *(x.grad.float() for x in (q, k, v))]) | |
| for x in (q, k, v): | |
| x.grad = None | |
| errors = [ | |
| ((a - b).norm() / b.norm()).item() for a, b in zip(*results, strict=True) | |
| ] | |
| print({"positions": positions, "relative_errors": errors}, flush=True) | |
| if max(errors) > 0.01: | |
| raise SystemExit("kernel differs from reference") | |