areneau's picture
Update Clairic Transformers runtime
24191a8 verified
Raw History Blame Contribute Delete
33.7 kB
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
# Copyright (c) 2024, Tri Dao.
# Copyright 2026 Cloudflare, Inc.
"""CUDA kernels for attention and causal convolution."""
from typing import Any
import torch
import triton
import triton.language as tl
@triton.jit(do_not_specialize=["XB", "XD", "XT", "OB", "OD", "OT"])
def _prefill(X: tl.tensor, W: tl.tensor, Bias: tl.tensor, Out: tl.tensor,
T: tl.int32, D: tl.constexpr,
XB: tl.int64, XD: tl.int64, XT: tl.int64,
OB: tl.int64, OD: tl.int64, OT: tl.int64,
HAS_BIAS: tl.constexpr, SILU: tl.constexpr,
BT: tl.constexpr, BD: tl.constexpr) -> None:
batch = tl.program_id(1)
token = tl.program_id(0) * BT + tl.arange(0, BT)
channel = tl.program_id(2) * BD + tl.arange(0, BD)
if HAS_BIAS:
initial = tl.load(Bias + channel, channel < D, other=0).to(tl.float32)
else:
initial = tl.full((BD,), 0, tl.float32)
acc = tl.broadcast_to(initial[None, :], (BT, BD))
for tap in tl.static_range(4):
position = token + tap - 3
value = tl.load(X + batch * XB + channel[None, :] * XD + position[:, None] * XT,
(position[:, None] >= 0) & (position[:, None] < T)
& (channel[None, :] < D), other=0)
weight = tl.load(W + channel * 4 + tap, channel < D, other=0)
acc += value * weight[None, :]
if SILU:
acc = acc / (1 + tl.exp(-acc))
tl.store(Out + batch * OB + channel[None, :] * OD + token[:, None] * OT,
acc, (token[:, None] < T) & (channel[None, :] < D))
@triton.jit(do_not_specialize=["XB", "XD", "SB", "SD", "ST", "OB", "OD"])
def _update(X: tl.tensor, State: tl.tensor, W: tl.tensor, Bias: tl.tensor, Out: tl.tensor,
D: tl.constexpr, XB: tl.int64, XD: tl.int64,
SB: tl.int64, SD: tl.int64, ST: tl.int64,
OB: tl.int64, OD: tl.int64,
HAS_BIAS: tl.constexpr, SILU: tl.constexpr, BD: tl.constexpr) -> None:
batch = tl.program_id(0)
channel = tl.program_id(1) * BD + tl.arange(0, BD)
mask = channel < D
base = State + batch * SB + channel * SD
x0 = tl.load(base + ST, mask, other=0)
x1 = tl.load(base + 2 * ST, mask, other=0)
x2 = tl.load(base + 3 * ST, mask, other=0)
x3 = tl.load(X + batch * XB + channel * XD, mask, other=0)
if HAS_BIAS:
acc = tl.load(Bias + channel, mask, other=0).to(tl.float32)
else:
acc = tl.full((BD,), 0, tl.float32)
for tap in tl.static_range(4):
if tap == 0:
value = x0
elif tap == 1:
value = x1
elif tap == 2:
value = x2
else:
value = x3
weight = tl.load(W + channel * 4 + tap, mask, other=0)
acc += value * weight
if SILU:
acc = acc / (1 + tl.exp(-acc))
tl.store(Out + batch * OB + channel * OD, acc, mask)
# Preserve the four-token cache owned by Transformers.
tl.store(base, x0, mask)
tl.store(base + ST, x1, mask)
tl.store(base + 2 * ST, x2, mask)
tl.store(base + 3 * ST, x3, mask)
def prefill(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None,
activation: str | None) -> torch.Tensor:
output = torch.empty_like(x)
batch, channels, length = x.shape
_prefill[(triton.cdiv(length, 8), batch, triton.cdiv(channels, 256))](
x, weight, bias, output, length, channels, *x.stride(), *output.stride(),
HAS_BIAS=bias is not None, SILU=activation is not None, BT=8, BD=256,
num_warps=4, num_stages=2,
)
return output
def update(x: torch.Tensor, state: torch.Tensor, weight: torch.Tensor,
bias: torch.Tensor | None, activation: str | None) -> torch.Tensor:
output = torch.empty_like(x)
_update[(x.shape[0], triton.cdiv(x.shape[1], 256))](
x, state, weight, bias, output, x.shape[1], *x.stride()[:2],
*state.stride(), *output.stride()[:2], HAS_BIAS=bias is not None,
SILU=activation is not None, BD=256, num_warps=4,
)
return output
@triton.jit(do_not_specialize=["QGT", "KT", "QT", "KOT", "GT", "CT", "ST"])
def _dense_qk_kernel(QG: tl.tensor, K: tl.tensor, QO: tl.tensor, KO: tl.tensor, GO: tl.tensor,
QW: tl.tensor, KW: tl.tensor, C: tl.tensor, S: tl.tensor,
QGT: tl.int64, KT: tl.int64, QT: tl.int64, KOT: tl.int64, GT: tl.int64,
CT: tl.int64, ST: tl.int64,
NQ: tl.constexpr, NKV: tl.constexpr, D: tl.constexpr, R: tl.constexpr,
HALF: tl.constexpr, HEAD_BLOCK: tl.constexpr, ROT_BLOCK: tl.constexpr,
EPS: tl.constexpr) -> None:
token, head = tl.program_id(0), tl.program_id(1)
is_k = head >= NQ
local_head = tl.where(is_k, head - NQ, head)
if is_k:
in_base = K + token * KT + local_head * D
w_ptr = KW
out_base = KO + token * KOT + local_head * D
else:
in_base = QG + token * QGT + local_head * 2 * D
w_ptr = QW
out_base = QO + token * QT + local_head * D
head_offs = tl.arange(0, HEAD_BLOCK)
head_mask = head_offs < D
x = tl.load(in_base + head_offs, mask=head_mask, other=0.0).to(tl.float32)
w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32)
var = tl.sum(x * x, axis=0) / D
inv_rms = tl.rsqrt(var + EPS)
x_norm = (x * inv_rms * (w + 1.0)).to(tl.bfloat16).to(tl.float32)
pass_mask = head_mask & (head_offs >= R)
tl.store(out_base + head_offs, x_norm, mask=pass_mask)
rot_offs = tl.arange(0, ROT_BLOCK)
rot_mask = rot_offs < HALF
xr1 = tl.load(in_base + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
xr2 = tl.load(in_base + HALF + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
wr1 = tl.load(w_ptr + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
wr2 = tl.load(w_ptr + HALF + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
xr1 = (xr1 * inv_rms * (wr1 + 1.0)).to(tl.bfloat16).to(tl.float32)
xr2 = (xr2 * inv_rms * (wr2 + 1.0)).to(tl.bfloat16).to(tl.float32)
cos = tl.load(C + token * CT + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
sin = tl.load(S + token * ST + rot_offs, mask=rot_mask, other=0.0).to(tl.float32)
tl.store(out_base + rot_offs, xr1 * cos - xr2 * sin, mask=rot_mask)
tl.store(out_base + HALF + rot_offs, xr2 * cos + xr1 * sin, mask=rot_mask)
gate_in = in_base + D
if not is_k:
gate_out = GO + token * GT + local_head * D
gate = tl.load(gate_in + head_offs, mask=head_mask, other=0.0)
tl.store(gate_out + head_offs, gate, mask=head_mask)
def dense_qk(q_gate: torch.Tensor, key: torch.Tensor, q_weight: torch.Tensor,
k_weight: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor,
eps: float) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
batch, length, q_width = q_gate.shape
dim, rotary_dim = q_weight.numel(), cos.shape[-1]
q_heads, kv_heads = q_width // (2 * dim), key.shape[-1] // dim
tokens = batch * length
qg, k = q_gate.view(tokens, -1), key.view(tokens, -1)
c, s = cos.view(tokens, rotary_dim), sin.view(tokens, rotary_dim)
q_out = torch.empty(tokens, q_heads * dim, device=q_gate.device, dtype=q_gate.dtype)
k_out = torch.empty(tokens, kv_heads * dim, device=key.device, dtype=key.dtype)
gate = torch.empty(tokens, q_heads * dim, device=q_gate.device, dtype=q_gate.dtype)
_dense_qk_kernel[(tokens, q_heads + kv_heads)](
qg, k, q_out, k_out, gate, q_weight, k_weight, c, s,
qg.stride(0), k.stride(0), q_out.stride(0), k_out.stride(0), gate.stride(0),
c.stride(0), s.stride(0), NQ=q_heads, NKV=kv_heads, D=dim, R=rotary_dim,
HALF=rotary_dim // 2, HEAD_BLOCK=triton.next_power_of_2(dim),
ROT_BLOCK=triton.next_power_of_2(rotary_dim // 2), EPS=eps, num_warps=4)
return (q_out.view(batch, length, q_heads, dim).transpose(1, 2),
k_out.view(batch, length, kv_heads, dim).transpose(1, 2),
gate.view(batch, length, q_heads * dim))
@triton.jit
def _layer_norm_fwd_1pass_kernel(
X: tl.tensor, Y: tl.tensor, W: tl.tensor, B: tl.tensor, Z: tl.tensor,
Mean: tl.tensor, Rstd: tl.tensor,
stride_x_row: tl.int64, stride_y_row: tl.int64, stride_z_row: tl.int64,
M: tl.int32, N: tl.constexpr, eps: tl.float32,
BLOCK_N: tl.constexpr, ROWS_PER_BLOCK: tl.constexpr,
HAS_BIAS: tl.constexpr, HAS_Z: tl.constexpr, NORM_BEFORE_GATE: tl.constexpr,
IS_RMS_NORM: tl.constexpr, ACTIVATION: tl.constexpr,
USE_GDC: tl.constexpr = False,
) -> None:
if USE_GDC:
tl.extra.cuda.gdc_wait()
row_start = tl.program_id(0) * ROWS_PER_BLOCK
group = tl.program_id(1)
rows = row_start + tl.arange(0, ROWS_PER_BLOCK)
cols = tl.arange(0, BLOCK_N)
row_offsets = rows[:, None] * stride_x_row
col_offsets = cols[None, :] + group * N
X_base = X + row_offsets + col_offsets
Y_base = Y + rows[:, None] * stride_y_row + col_offsets
row_mask = rows[:, None] < M
col_mask = cols[None, :] < N
mask = row_mask & col_mask
x = tl.load(X_base, mask=mask, other=0.0).to(tl.float32)
if HAS_Z and (not NORM_BEFORE_GATE):
Z_base = Z + rows[:, None] * stride_z_row + col_offsets
z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32)
if ACTIVATION == 'swish' or ACTIVATION == 'silu':
x *= z * tl.sigmoid(z)
elif ACTIVATION == 'sigmoid':
x *= tl.sigmoid(z)
if not IS_RMS_NORM:
mean = tl.sum(x, axis=1) / N
mean_offsets = group * M + rows
mean_mask = rows < M
tl.store(Mean + mean_offsets, mean, mask=mean_mask)
xbar = tl.where(mask, x - mean[:, None], 0.0)
var = tl.sum(xbar * xbar, axis=1) / N
else:
xbar = tl.where(mask, x, 0.0)
var = tl.sum(xbar * xbar, axis=1) / N
mean = 0.0
rstd = tl.rsqrt(var + eps)
rstd_offsets = group * M + rows
rstd_mask = rows < M
tl.store(Rstd + rstd_offsets, rstd, mask=rstd_mask)
w_offsets = cols + group * N
w_mask = cols < N
w = tl.load(W + w_offsets, mask=w_mask, other=0.0).to(tl.float32)
if HAS_BIAS:
b = tl.load(B + w_offsets, mask=w_mask, other=0.0).to(tl.float32)
if not IS_RMS_NORM:
x_hat = (x - mean[:, None]) * rstd[:, None]
else:
x_hat = x * rstd[:, None]
y = x_hat * w[None, :] + b[None, :] if HAS_BIAS else x_hat * w[None, :]
if HAS_Z and NORM_BEFORE_GATE:
Z_base = Z + rows[:, None] * stride_z_row + col_offsets
z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32)
if ACTIVATION == 'swish' or ACTIVATION == 'silu':
y *= z * tl.sigmoid(z)
elif ACTIVATION == 'sigmoid':
y *= tl.sigmoid(z)
tl.store(Y_base, y, mask=mask)
if USE_GDC:
tl.extra.cuda.gdc_launch_dependents()
def gated_norm(x: torch.Tensor, z: torch.Tensor, weight: torch.Tensor,
eps: float) -> torch.Tensor:
output = torch.empty_like(x)
rstd = torch.empty((x.shape[0],), dtype=torch.float32, device=x.device)
block_n = triton.next_power_of_2(x.shape[1])
pdl = torch.cuda.get_device_capability(x.device)[0] >= 9
launch = {"USE_GDC": True, "launch_pdl": True} if pdl else {}
_layer_norm_fwd_1pass_kernel[(triton.cdiv(x.shape[0], 4), 1)](
x, output, weight, None, z, None, rstd,
x.stride(0), output.stride(0), z.stride(0), x.shape[0], x.shape[1], eps,
BLOCK_N=block_n, ROWS_PER_BLOCK=4, HAS_BIAS=False, HAS_Z=True,
NORM_BEFORE_GATE=True, IS_RMS_NORM=True, ACTIVATION="swish",
num_warps=min(max(block_n // 256, 1), 8), **launch,
)
return output
@triton.jit
def _fused_sigmoid_mul_kernel(
output_ptr: tl.tensor, attn_output_ptr: tl.tensor, gate_ptr: tl.tensor,
gate_stride_row: tl.int64, gate_stride_head: tl.int64,
hidden_dim: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_H: tl.constexpr,
) -> None:
"""Fuse sigmoid(gate) * attn_output into a single kernel."""
pid_row = tl.program_id(0).to(tl.int64)
pid_block = tl.program_id(1)
offsets = pid_block * BLOCK_H + tl.arange(0, BLOCK_H)
mask = offsets < hidden_dim
head = offsets // HEAD_DIM
d = offsets - head * HEAD_DIM
attn_off = pid_row * hidden_dim + offsets
attn = tl.load(attn_output_ptr + attn_off, mask=mask, other=0.0).to(tl.float32)
gate_off = pid_row * gate_stride_row + head * gate_stride_head + d
g = tl.load(gate_ptr + gate_off, mask=mask, other=0.0).to(tl.float32)
result = attn * tl.sigmoid(g)
tl.store(output_ptr + attn_off, result, mask=mask)
def fused_sigmoid_mul(attn_output: torch.Tensor, gate: torch.Tensor, inplace: bool=False) -> torch.Tensor:
"""
Fused sigmoid-mul for attention output gating.
Equivalent to: attn_output * sigmoid(gate)
The production Qwen3.5 path passes a 3D strided gate. A single hidden-block
Triton kernel handles both that path and flat contiguous inputs.
When inplace=True, writes result back to attn_output and returns it.
Supports strided gate: if gate is 3D (num_tokens, num_heads, head_dim)
and attn_output is 2D (num_tokens, hidden_dim), the kernel reads gate
via explicit strides without requiring a contiguous copy.
"""
if gate.ndim == 3 and attn_output.ndim == 2:
num_tokens, num_heads, head_dim = gate.shape
hidden_dim = num_heads * head_dim
assert attn_output.shape == (num_tokens, hidden_dim)
gate_stride_row = gate.stride(0)
gate_stride_head = gate.stride(1)
else:
assert attn_output.shape == gate.shape, 'attn_output and gate must have the same shape'
hidden_dim = attn_output.shape[-1]
num_tokens = attn_output.numel() // hidden_dim
head_dim = hidden_dim
gate_stride_row = hidden_dim
gate_stride_head = hidden_dim
out = attn_output if inplace else torch.empty_like(attn_output)
block_h = 1024 if num_tokens < 1024 else 2048
grid = (num_tokens, triton.cdiv(hidden_dim, block_h))
_fused_sigmoid_mul_kernel[grid](out, attn_output, gate, gate_stride_row, gate_stride_head, hidden_dim, HEAD_DIM=head_dim, BLOCK_H=block_h, num_warps=4)
return out
@triton.jit
def _silu_mul_kernel(
gate_ptr: tl.tensor, up_ptr: tl.tensor, output_ptr: tl.tensor,
elements: tl.constexpr, BLOCK: tl.constexpr,
) -> None:
offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
mask = offsets < elements
gate = tl.load(gate_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
up = tl.load(up_ptr + offsets, mask=mask, other=0.0).to(tl.float32)
# Preserve the qualified CUDA --use_fast_math operations, including FTZ.
output = tl.inline_asm_elementwise(
"{ .reg .f32 exponent, denominator, activated; "
"mul.ftz.f32 exponent, $1, 0fBFB8AA3B; "
"ex2.approx.ftz.f32 exponent, exponent; "
"add.ftz.f32 denominator, exponent, 0f3F800000; "
"div.approx.ftz.f32 activated, $1, denominator; "
"mul.ftz.f32 $0, $2, activated; }",
constraints="=f,f,f", args=[gate, up], dtype=tl.float32,
is_pure=True, pack=1,
)
tl.store(output_ptr + offsets, output, mask=mask)
def silu_mul(gate: torch.Tensor, up: torch.Tensor) -> torch.Tensor:
"""Match the qualified fused CUDA SiLU-mul before the BF16 output store."""
if (gate.shape != up.shape or gate.device != up.device
or gate.dtype != torch.bfloat16 or up.dtype != torch.bfloat16
or not gate.is_cuda or not gate.is_contiguous() or not up.is_contiguous()):
raise ValueError("Expected aligned contiguous CUDA BF16 gate and up tensors")
output = torch.empty_like(gate)
_silu_mul_kernel[(triton.cdiv(gate.numel(), 2048),)](
gate, up, output, gate.numel(), BLOCK=2048, num_warps=8,
)
return output
# MIT License
#
# Copyright 2025 Thinking Machines Lab
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# The above copyright notice and this permission notice shall be included in all
# copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
def get_device_core_count() -> int:
return torch.cuda.get_device_properties(torch.cuda.current_device()).multi_processor_count
@triton.jit
def _compute_pid(tile_id: tl.tensor, num_pid_in_group: tl.tensor, num_pid_m: tl.tensor, GROUP_SIZE_M: tl.tensor, NUM_SMS: tl.tensor) -> tuple[tl.tensor, tl.tensor]:
group_id = tile_id // num_pid_in_group
first_pid_m = group_id * GROUP_SIZE_M
group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M)
pid_m = first_pid_m + tile_id % group_size_m
pid_n = tile_id % num_pid_in_group // group_size_m
return (pid_m, pid_n)
@triton.jit
def matmul_kernel_persistent(a_ptr: tl.tensor, b_ptr: tl.tensor, c_ptr: tl.tensor, bias_ptr: tl.tensor, M: tl.int32, N: tl.int32, K: tl.int32, stride_am: tl.int32, stride_ak: tl.int32, stride_bk: tl.int32, stride_bn: tl.int32, stride_cm: tl.int32, stride_cn: tl.int32, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, NUM_SMS: tl.constexpr, A_LARGE: tl.constexpr, B_LARGE: tl.constexpr, C_LARGE: tl.constexpr, HAS_BIAS: tl.constexpr) -> None:
start_pid = tl.program_id(axis=0)
num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
k_tiles = tl.cdiv(K, BLOCK_SIZE_K)
num_tiles = num_pid_m * num_pid_n
offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K)
num_pid_in_group = GROUP_SIZE_M * num_pid_n
for tile_id in tl.range(start_pid, num_tiles, NUM_SMS, flatten=True):
(pid_m, pid_n) = _compute_pid(tile_id, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS)
start_m = pid_m * BLOCK_SIZE_M
start_n = pid_n * BLOCK_SIZE_N
offs_am = start_m + tl.arange(0, BLOCK_SIZE_M)
offs_bn = start_n + tl.arange(0, BLOCK_SIZE_N)
if A_LARGE:
offs_am = offs_am.to(tl.int64)
if B_LARGE:
offs_bn = offs_bn.to(tl.int64)
offs_am = tl.where(offs_am < M, offs_am, 0)
offs_bn = tl.where(offs_bn < N, offs_bn, 0)
offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M)
offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N)
accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
for ki in range(k_tiles):
if A_LARGE or B_LARGE:
offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K).to(tl.int64)
else:
offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K)
a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak)
b_ptrs = b_ptr + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
a = tl.load(a_ptrs, mask=offs_k_for_mask[None, :] < K - ki * BLOCK_SIZE_K, other=0.0)
b = tl.load(b_ptrs, mask=offs_k_for_mask[:, None] < K - ki * BLOCK_SIZE_K, other=0.0)
accumulator = tl.dot(a, b, accumulator)
offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
if C_LARGE:
offs_cm = offs_cm.to(tl.int64)
offs_cn = offs_cn.to(tl.int64)
c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
if HAS_BIAS:
bias_ptrs = bias_ptr + offs_cn
bias = tl.load(bias_ptrs, mask=offs_cn < N, other=0.0).to(tl.float32)
accumulator += bias
if c_ptr.dtype.element_ty == tl.float8e4nv:
c = accumulator.to(tl.float8e4nv)
elif c_ptr.dtype.element_ty == tl.bfloat16:
c = accumulator.to(tl.bfloat16)
elif c_ptr.dtype.element_ty == tl.float32:
c = accumulator.to(tl.float32)
else:
c = accumulator.to(tl.float16)
tl.store(c_ptrs, c, mask=c_mask)
def _matmul_persistent_triton(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None=None) -> torch.Tensor:
assert a.shape[1] == b.shape[0], 'Incompatible dimensions'
assert a.dtype == b.dtype, 'Incompatible dtypes'
assert bias is None or bias.dim() == 1, 'Expected a one-dimensional bias'
NUM_SMS = get_device_core_count()
(M, K) = a.shape
(K, N) = b.shape
dtype = a.dtype
c = torch.empty((M, N), device=a.device, dtype=dtype)
def grid(META: dict[str, Any]) -> tuple[int]:
return (min(NUM_SMS, triton.cdiv(M, META['BLOCK_SIZE_M']) * triton.cdiv(N, META['BLOCK_SIZE_N'])),)
configs = {torch.bfloat16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float32: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}}
matmul_kernel_persistent[grid](a, b, c, bias, M, N, K, a.stride(0), a.stride(1), b.stride(0), b.stride(1), c.stride(0), c.stride(1), NUM_SMS=NUM_SMS, A_LARGE=a.numel() > 2 ** 31, B_LARGE=b.numel() > 2 ** 31, C_LARGE=c.numel() > 2 ** 31, HAS_BIAS=bias is not None, **configs[dtype])
return c
@triton.jit
def mean_kernel(input_ptr: tl.tensor, output_ptr: tl.tensor, input_stride0: tl.int32, input_stride1: tl.int32, input_stride2: tl.int32, output_stride0: tl.int32, output_stride1: tl.int32, M: tl.int32, N: tl.int32, K: tl.int32, BLOCK_SIZE: tl.constexpr) -> None:
"""
Kernel for computing mean along a single dimension.
Input is viewed as (M, N, K) where N is the dimension being reduced.
"""
pid = tl.program_id(0)
m_idx = pid // K
k_idx = pid % K
if m_idx >= M or k_idx >= K:
return
acc = 0.0
for n_start in range(0, N, BLOCK_SIZE):
n_offsets = n_start + tl.arange(0, BLOCK_SIZE)
mask = n_offsets < N
input_idx = m_idx * input_stride0 + n_offsets * input_stride1 + k_idx * input_stride2
vals = tl.load(input_ptr + input_idx, mask=mask, other=0.0)
acc += tl.sum(vals)
mean_val = acc / N
output_idx = m_idx * output_stride0 + k_idx * output_stride1
tl.store(output_ptr + output_idx, mean_val)
def mean_dim(input: torch.Tensor, dim: int, keepdim: bool=False, dtype: torch.dtype | None=None) -> torch.Tensor:
"""
Triton implementation of torch.mean with single dimension reduction.
Args:
input: Input tensor
dim: Single dimension along which to compute mean
keepdim: Whether to keep the reduced dimension
dtype: Output dtype. If None, uses input dtype (or float32 for integer inputs)
Returns:
Tensor with mean values along specified dimension
"""
assert input.is_cuda or input.is_xpu, 'Input must be a CUDA or XPU tensor'
assert -input.ndim <= dim < input.ndim, f'Invalid dimension {dim} for tensor with {input.ndim} dimensions'
if dim < 0:
dim = dim + input.ndim
if dtype is None:
if input.dtype in [torch.int8, torch.int16, torch.int32, torch.int64]:
dtype = torch.float32
else:
dtype = input.dtype
if input.dtype != dtype:
input = input.to(dtype)
shape = list(input.shape)
M = 1
for i in range(dim):
M *= shape[i]
N = shape[dim]
K = 1
for i in range(dim + 1, len(shape)):
K *= shape[i]
input_3d = input.reshape(M, N, K)
if keepdim:
output_shape = shape.copy()
output_shape[dim] = 1
else:
output_shape = shape[:dim] + shape[dim + 1:]
output = torch.empty(output_shape, dtype=dtype, device=input.device)
if keepdim:
output_2d = output.reshape(M, 1, K).squeeze(1)
else:
output_2d = output.reshape(M, K)
grid = (M * K,)
BLOCK_SIZE = 1024
mean_kernel[grid](input_3d, output_2d, input_3d.stride(0), input_3d.stride(1), input_3d.stride(2), output_2d.stride(0), output_2d.stride(1) if output_2d.ndim > 1 else 0, M, N, K, BLOCK_SIZE)
return output
def mean_batch_invariant(input: torch.Tensor, dim: list[int], keepdim: bool=False, dtype: torch.dtype | None=None) -> torch.Tensor:
assert dtype is None or dtype == torch.float32, f'unsupported dtype: {dtype}'
if len(dim) == 1:
return mean_dim(input, dim[0], keepdim=keepdim)
else:
assert input.dtype in {torch.float16, torch.bfloat16, torch.float32}, 'only float types supported for now'
n_elems = 1
for d in dim:
n_elems *= input.shape[d]
return torch.sum(input, dim=dim, keepdim=keepdim, dtype=torch.float32) / n_elems
@triton.jit
def bmm_kernel_persistent(a_ptr: tl.tensor, b_ptr: tl.tensor, c_ptr: tl.tensor, B: tl.int32, M: tl.int32, N: tl.int32, K: tl.int32, stride_ab: tl.int32, stride_am: tl.int32, stride_ak: tl.int32, stride_bb: tl.int32, stride_bk: tl.int32, stride_bn: tl.int32, stride_cb: tl.int32, stride_cm: tl.int32, stride_cn: tl.int32, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, NUM_SMS: tl.constexpr, A_LARGE: tl.constexpr, B_LARGE: tl.constexpr, C_LARGE: tl.constexpr) -> None:
"""
Batched matrix multiplication kernel that processes batches in parallel.
Each tile processes a (BLOCK_SIZE_M, BLOCK_SIZE_N) output block for a specific batch.
"""
start_pid = tl.program_id(axis=0)
num_pid_m = tl.cdiv(M, BLOCK_SIZE_M)
num_pid_n = tl.cdiv(N, BLOCK_SIZE_N)
k_tiles = tl.cdiv(K, BLOCK_SIZE_K)
num_tiles_per_batch = num_pid_m * num_pid_n
num_tiles_total = B * num_tiles_per_batch
offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K)
num_pid_in_group = GROUP_SIZE_M * num_pid_n
for tile_id in tl.range(start_pid, num_tiles_total, NUM_SMS, flatten=True):
batch_idx = tile_id // num_tiles_per_batch
tile_in_batch = tile_id % num_tiles_per_batch
(pid_m, pid_n) = _compute_pid(tile_in_batch, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS)
start_m = pid_m * BLOCK_SIZE_M
start_n = pid_n * BLOCK_SIZE_N
offs_am = start_m + tl.arange(0, BLOCK_SIZE_M)
offs_bn = start_n + tl.arange(0, BLOCK_SIZE_N)
if A_LARGE:
offs_am = offs_am.to(tl.int64)
if B_LARGE:
offs_bn = offs_bn.to(tl.int64)
offs_am = tl.where(offs_am < M, offs_am, 0)
offs_bn = tl.where(offs_bn < N, offs_bn, 0)
offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M)
offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N)
if A_LARGE or B_LARGE:
batch_idx_typed = batch_idx.to(tl.int64)
else:
batch_idx_typed = batch_idx
accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32)
for ki in range(k_tiles):
if A_LARGE or B_LARGE:
offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K).to(tl.int64)
else:
offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K)
a_ptrs = a_ptr + (batch_idx_typed * stride_ab + offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak)
b_ptrs = b_ptr + (batch_idx_typed * stride_bb + offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn)
a = tl.load(a_ptrs, mask=offs_k_for_mask[None, :] < K - ki * BLOCK_SIZE_K, other=0.0)
b = tl.load(b_ptrs, mask=offs_k_for_mask[:, None] < K - ki * BLOCK_SIZE_K, other=0.0)
accumulator = tl.dot(a, b, accumulator)
offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M)
offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N)
if C_LARGE:
offs_cm = offs_cm.to(tl.int64)
offs_cn = offs_cn.to(tl.int64)
c_ptrs = c_ptr + batch_idx_typed * stride_cb + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :]
c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N)
if c_ptr.dtype.element_ty == tl.float8e4nv:
c = accumulator.to(tl.float8e4nv)
elif c_ptr.dtype.element_ty == tl.bfloat16:
c = accumulator.to(tl.bfloat16)
elif c_ptr.dtype.element_ty == tl.float32:
c = accumulator.to(tl.float32)
else:
c = accumulator.to(tl.float16)
tl.store(c_ptrs, c, mask=c_mask)
def bmm_batch_invariant(a: torch.Tensor, b: torch.Tensor, *, out: torch.Tensor | None=None) -> torch.Tensor:
if a.ndim == 3 and b.ndim == 3:
assert a.shape[0] == b.shape[0], 'Batch sizes must match'
assert a.shape[2] == b.shape[1], 'Incompatible dimensions'
assert a.dtype == b.dtype, 'Incompatible dtypes'
B = a.shape[0]
M = a.shape[1]
K = a.shape[2]
N = b.shape[2]
dtype = a.dtype
if out is None:
c = torch.empty((B, M, N), device=a.device, dtype=dtype)
else:
c = out
NUM_SMS = get_device_core_count()
configs = {torch.bfloat16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float32: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}}
config = configs.get(dtype)
if config is None:
raise ValueError(f'Unsupported dtype {dtype} for bmm_batch_invariant. Supported dtypes are: {list(configs.keys())}')
num_tiles_per_batch = triton.cdiv(M, config['BLOCK_SIZE_M']) * triton.cdiv(N, config['BLOCK_SIZE_N'])
num_tiles_total = B * num_tiles_per_batch
grid = (min(NUM_SMS, num_tiles_total),)
bmm_kernel_persistent[grid](a, b, c, B, M, N, K, a.stride(0), a.stride(1), a.stride(2), b.stride(0), b.stride(1), b.stride(2), c.stride(0), c.stride(1), c.stride(2), NUM_SMS=NUM_SMS, A_LARGE=a.numel() > 2 ** 31, B_LARGE=b.numel() > 2 ** 31, C_LARGE=c.numel() > 2 ** 31, **config)
return c
else:
raise ValueError(f'bmm_batch_invariant expects 3D tensors, got shapes {a.shape} and {b.shape}')
@triton.jit(do_not_specialize=["count"])
def _bf16_pair_kernel(value: tl.tensor, high: tl.tensor, low: tl.tensor,
count: tl.int32, limit: tl.constexpr, block: tl.constexpr) -> None:
offsets = tl.program_id(0) * block + tl.arange(0, block)
current = tl.load(value + offsets, offsets < count, other=0)
clipped = tl.minimum(tl.maximum(current, -limit, propagate_nan=tl.PropagateNan.ALL),
limit, propagate_nan=tl.PropagateNan.ALL)
upper = clipped.to(tl.bfloat16)
lower = (current - upper.to(tl.float32)).to(tl.bfloat16)
lower = tl.where(current == 0, upper, lower)
tl.store(high + offsets, upper, offsets < count)
tl.store(low + offsets, lower, offsets < count)
def bf16_pair_sum(value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
if not value.is_cuda or value.dtype != torch.float32 or not value.is_contiguous():
raise ValueError("Expected contiguous CUDA FP32 residuals")
high = torch.empty_like(value, dtype=torch.bfloat16)
low = torch.empty_like(value, dtype=torch.bfloat16)
count = value.numel()
if count:
_bf16_pair_kernel[(triton.cdiv(count, 1024),)](
value, high, low, count, torch.finfo(torch.bfloat16).max, 1024,
num_warps=4, enable_fp_fusion=False,
)
return high, low
@triton.jit(do_not_specialize=["count"])
def _residual_kernel(first: tl.tensor, second: tl.tensor, output: tl.tensor,
count: tl.int32, block: tl.constexpr) -> None:
offsets = tl.program_id(0) * block + tl.arange(0, block)
upper = tl.load(first + offsets, offsets < count, other=0).to(tl.bfloat16).to(tl.float32)
lower = tl.load(second + offsets, offsets < count, other=0).to(tl.float32)
tl.store(output + offsets, upper + lower, offsets < count)
def residual_sum(first: torch.Tensor, second: torch.Tensor) -> torch.Tensor:
if (not first.is_cuda or second.device != first.device or second.shape != first.shape
or first.dtype not in (torch.bfloat16, torch.float32)
or second.dtype != torch.bfloat16
or not first.is_contiguous() or not second.is_contiguous()):
raise ValueError("Expected aligned contiguous CUDA residuals")
output = torch.empty_like(first, dtype=torch.float32)
count = first.numel()
if count:
_residual_kernel[(triton.cdiv(count, 1024),)](
first, second, output, count, 1024, num_warps=4, enable_fp_fusion=False,
)
return output