# SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project # Copyright (c) 2024, Tri Dao. # Copyright 2026 Cloudflare, Inc. """CUDA kernels for attention and causal convolution.""" from typing import Any import torch import triton import triton.language as tl @triton.jit(do_not_specialize=["XB", "XD", "XT", "OB", "OD", "OT"]) def _prefill(X: tl.tensor, W: tl.tensor, Bias: tl.tensor, Out: tl.tensor, T: tl.int32, D: tl.constexpr, XB: tl.int64, XD: tl.int64, XT: tl.int64, OB: tl.int64, OD: tl.int64, OT: tl.int64, HAS_BIAS: tl.constexpr, SILU: tl.constexpr, BT: tl.constexpr, BD: tl.constexpr) -> None: batch = tl.program_id(1) token = tl.program_id(0) * BT + tl.arange(0, BT) channel = tl.program_id(2) * BD + tl.arange(0, BD) if HAS_BIAS: initial = tl.load(Bias + channel, channel < D, other=0).to(tl.float32) else: initial = tl.full((BD,), 0, tl.float32) acc = tl.broadcast_to(initial[None, :], (BT, BD)) for tap in tl.static_range(4): position = token + tap - 3 value = tl.load(X + batch * XB + channel[None, :] * XD + position[:, None] * XT, (position[:, None] >= 0) & (position[:, None] < T) & (channel[None, :] < D), other=0) weight = tl.load(W + channel * 4 + tap, channel < D, other=0) acc += value * weight[None, :] if SILU: acc = acc / (1 + tl.exp(-acc)) tl.store(Out + batch * OB + channel[None, :] * OD + token[:, None] * OT, acc, (token[:, None] < T) & (channel[None, :] < D)) @triton.jit(do_not_specialize=["XB", "XD", "SB", "SD", "ST", "OB", "OD"]) def _update(X: tl.tensor, State: tl.tensor, W: tl.tensor, Bias: tl.tensor, Out: tl.tensor, D: tl.constexpr, XB: tl.int64, XD: tl.int64, SB: tl.int64, SD: tl.int64, ST: tl.int64, OB: tl.int64, OD: tl.int64, HAS_BIAS: tl.constexpr, SILU: tl.constexpr, BD: tl.constexpr) -> None: batch = tl.program_id(0) channel = tl.program_id(1) * BD + tl.arange(0, BD) mask = channel < D base = State + batch * SB + channel * SD x0 = tl.load(base + ST, mask, other=0) x1 = tl.load(base + 2 * ST, mask, other=0) x2 = tl.load(base + 3 * ST, mask, other=0) x3 = tl.load(X + batch * XB + channel * XD, mask, other=0) if HAS_BIAS: acc = tl.load(Bias + channel, mask, other=0).to(tl.float32) else: acc = tl.full((BD,), 0, tl.float32) for tap in tl.static_range(4): if tap == 0: value = x0 elif tap == 1: value = x1 elif tap == 2: value = x2 else: value = x3 weight = tl.load(W + channel * 4 + tap, mask, other=0) acc += value * weight if SILU: acc = acc / (1 + tl.exp(-acc)) tl.store(Out + batch * OB + channel * OD, acc, mask) # Preserve the four-token cache owned by Transformers. tl.store(base, x0, mask) tl.store(base + ST, x1, mask) tl.store(base + 2 * ST, x2, mask) tl.store(base + 3 * ST, x3, mask) def prefill(x: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None, activation: str | None) -> torch.Tensor: output = torch.empty_like(x) batch, channels, length = x.shape _prefill[(triton.cdiv(length, 8), batch, triton.cdiv(channels, 256))]( x, weight, bias, output, length, channels, *x.stride(), *output.stride(), HAS_BIAS=bias is not None, SILU=activation is not None, BT=8, BD=256, num_warps=4, num_stages=2, ) return output def update(x: torch.Tensor, state: torch.Tensor, weight: torch.Tensor, bias: torch.Tensor | None, activation: str | None) -> torch.Tensor: output = torch.empty_like(x) _update[(x.shape[0], triton.cdiv(x.shape[1], 256))]( x, state, weight, bias, output, x.shape[1], *x.stride()[:2], *state.stride(), *output.stride()[:2], HAS_BIAS=bias is not None, SILU=activation is not None, BD=256, num_warps=4, ) return output @triton.jit(do_not_specialize=["QGT", "KT", "QT", "KOT", "GT", "CT", "ST"]) def _dense_qk_kernel(QG: tl.tensor, K: tl.tensor, QO: tl.tensor, KO: tl.tensor, GO: tl.tensor, QW: tl.tensor, KW: tl.tensor, C: tl.tensor, S: tl.tensor, QGT: tl.int64, KT: tl.int64, QT: tl.int64, KOT: tl.int64, GT: tl.int64, CT: tl.int64, ST: tl.int64, NQ: tl.constexpr, NKV: tl.constexpr, D: tl.constexpr, R: tl.constexpr, HALF: tl.constexpr, HEAD_BLOCK: tl.constexpr, ROT_BLOCK: tl.constexpr, EPS: tl.constexpr) -> None: token, head = tl.program_id(0), tl.program_id(1) is_k = head >= NQ local_head = tl.where(is_k, head - NQ, head) if is_k: in_base = K + token * KT + local_head * D w_ptr = KW out_base = KO + token * KOT + local_head * D else: in_base = QG + token * QGT + local_head * 2 * D w_ptr = QW out_base = QO + token * QT + local_head * D head_offs = tl.arange(0, HEAD_BLOCK) head_mask = head_offs < D x = tl.load(in_base + head_offs, mask=head_mask, other=0.0).to(tl.float32) w = tl.load(w_ptr + head_offs, mask=head_mask, other=0.0).to(tl.float32) var = tl.sum(x * x, axis=0) / D inv_rms = tl.rsqrt(var + EPS) x_norm = (x * inv_rms * (w + 1.0)).to(tl.bfloat16).to(tl.float32) pass_mask = head_mask & (head_offs >= R) tl.store(out_base + head_offs, x_norm, mask=pass_mask) rot_offs = tl.arange(0, ROT_BLOCK) rot_mask = rot_offs < HALF xr1 = tl.load(in_base + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) xr2 = tl.load(in_base + HALF + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) wr1 = tl.load(w_ptr + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) wr2 = tl.load(w_ptr + HALF + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) xr1 = (xr1 * inv_rms * (wr1 + 1.0)).to(tl.bfloat16).to(tl.float32) xr2 = (xr2 * inv_rms * (wr2 + 1.0)).to(tl.bfloat16).to(tl.float32) cos = tl.load(C + token * CT + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) sin = tl.load(S + token * ST + rot_offs, mask=rot_mask, other=0.0).to(tl.float32) tl.store(out_base + rot_offs, xr1 * cos - xr2 * sin, mask=rot_mask) tl.store(out_base + HALF + rot_offs, xr2 * cos + xr1 * sin, mask=rot_mask) gate_in = in_base + D if not is_k: gate_out = GO + token * GT + local_head * D gate = tl.load(gate_in + head_offs, mask=head_mask, other=0.0) tl.store(gate_out + head_offs, gate, mask=head_mask) def dense_qk(q_gate: torch.Tensor, key: torch.Tensor, q_weight: torch.Tensor, k_weight: torch.Tensor, cos: torch.Tensor, sin: torch.Tensor, eps: float) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]: batch, length, q_width = q_gate.shape dim, rotary_dim = q_weight.numel(), cos.shape[-1] q_heads, kv_heads = q_width // (2 * dim), key.shape[-1] // dim tokens = batch * length qg, k = q_gate.view(tokens, -1), key.view(tokens, -1) c, s = cos.view(tokens, rotary_dim), sin.view(tokens, rotary_dim) q_out = torch.empty(tokens, q_heads * dim, device=q_gate.device, dtype=q_gate.dtype) k_out = torch.empty(tokens, kv_heads * dim, device=key.device, dtype=key.dtype) gate = torch.empty(tokens, q_heads * dim, device=q_gate.device, dtype=q_gate.dtype) _dense_qk_kernel[(tokens, q_heads + kv_heads)]( qg, k, q_out, k_out, gate, q_weight, k_weight, c, s, qg.stride(0), k.stride(0), q_out.stride(0), k_out.stride(0), gate.stride(0), c.stride(0), s.stride(0), NQ=q_heads, NKV=kv_heads, D=dim, R=rotary_dim, HALF=rotary_dim // 2, HEAD_BLOCK=triton.next_power_of_2(dim), ROT_BLOCK=triton.next_power_of_2(rotary_dim // 2), EPS=eps, num_warps=4) return (q_out.view(batch, length, q_heads, dim).transpose(1, 2), k_out.view(batch, length, kv_heads, dim).transpose(1, 2), gate.view(batch, length, q_heads * dim)) @triton.jit def _layer_norm_fwd_1pass_kernel( X: tl.tensor, Y: tl.tensor, W: tl.tensor, B: tl.tensor, Z: tl.tensor, Mean: tl.tensor, Rstd: tl.tensor, stride_x_row: tl.int64, stride_y_row: tl.int64, stride_z_row: tl.int64, M: tl.int32, N: tl.constexpr, eps: tl.float32, BLOCK_N: tl.constexpr, ROWS_PER_BLOCK: tl.constexpr, HAS_BIAS: tl.constexpr, HAS_Z: tl.constexpr, NORM_BEFORE_GATE: tl.constexpr, IS_RMS_NORM: tl.constexpr, ACTIVATION: tl.constexpr, USE_GDC: tl.constexpr = False, ) -> None: if USE_GDC: tl.extra.cuda.gdc_wait() row_start = tl.program_id(0) * ROWS_PER_BLOCK group = tl.program_id(1) rows = row_start + tl.arange(0, ROWS_PER_BLOCK) cols = tl.arange(0, BLOCK_N) row_offsets = rows[:, None] * stride_x_row col_offsets = cols[None, :] + group * N X_base = X + row_offsets + col_offsets Y_base = Y + rows[:, None] * stride_y_row + col_offsets row_mask = rows[:, None] < M col_mask = cols[None, :] < N mask = row_mask & col_mask x = tl.load(X_base, mask=mask, other=0.0).to(tl.float32) if HAS_Z and (not NORM_BEFORE_GATE): Z_base = Z + rows[:, None] * stride_z_row + col_offsets z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32) if ACTIVATION == 'swish' or ACTIVATION == 'silu': x *= z * tl.sigmoid(z) elif ACTIVATION == 'sigmoid': x *= tl.sigmoid(z) if not IS_RMS_NORM: mean = tl.sum(x, axis=1) / N mean_offsets = group * M + rows mean_mask = rows < M tl.store(Mean + mean_offsets, mean, mask=mean_mask) xbar = tl.where(mask, x - mean[:, None], 0.0) var = tl.sum(xbar * xbar, axis=1) / N else: xbar = tl.where(mask, x, 0.0) var = tl.sum(xbar * xbar, axis=1) / N mean = 0.0 rstd = tl.rsqrt(var + eps) rstd_offsets = group * M + rows rstd_mask = rows < M tl.store(Rstd + rstd_offsets, rstd, mask=rstd_mask) w_offsets = cols + group * N w_mask = cols < N w = tl.load(W + w_offsets, mask=w_mask, other=0.0).to(tl.float32) if HAS_BIAS: b = tl.load(B + w_offsets, mask=w_mask, other=0.0).to(tl.float32) if not IS_RMS_NORM: x_hat = (x - mean[:, None]) * rstd[:, None] else: x_hat = x * rstd[:, None] y = x_hat * w[None, :] + b[None, :] if HAS_BIAS else x_hat * w[None, :] if HAS_Z and NORM_BEFORE_GATE: Z_base = Z + rows[:, None] * stride_z_row + col_offsets z = tl.load(Z_base, mask=mask, other=0.0).to(tl.float32) if ACTIVATION == 'swish' or ACTIVATION == 'silu': y *= z * tl.sigmoid(z) elif ACTIVATION == 'sigmoid': y *= tl.sigmoid(z) tl.store(Y_base, y, mask=mask) if USE_GDC: tl.extra.cuda.gdc_launch_dependents() def gated_norm(x: torch.Tensor, z: torch.Tensor, weight: torch.Tensor, eps: float) -> torch.Tensor: output = torch.empty_like(x) rstd = torch.empty((x.shape[0],), dtype=torch.float32, device=x.device) block_n = triton.next_power_of_2(x.shape[1]) pdl = torch.cuda.get_device_capability(x.device)[0] >= 9 launch = {"USE_GDC": True, "launch_pdl": True} if pdl else {} _layer_norm_fwd_1pass_kernel[(triton.cdiv(x.shape[0], 4), 1)]( x, output, weight, None, z, None, rstd, x.stride(0), output.stride(0), z.stride(0), x.shape[0], x.shape[1], eps, BLOCK_N=block_n, ROWS_PER_BLOCK=4, HAS_BIAS=False, HAS_Z=True, NORM_BEFORE_GATE=True, IS_RMS_NORM=True, ACTIVATION="swish", num_warps=min(max(block_n // 256, 1), 8), **launch, ) return output @triton.jit def _fused_sigmoid_mul_kernel( output_ptr: tl.tensor, attn_output_ptr: tl.tensor, gate_ptr: tl.tensor, gate_stride_row: tl.int64, gate_stride_head: tl.int64, hidden_dim: tl.constexpr, HEAD_DIM: tl.constexpr, BLOCK_H: tl.constexpr, ) -> None: """Fuse sigmoid(gate) * attn_output into a single kernel.""" pid_row = tl.program_id(0).to(tl.int64) pid_block = tl.program_id(1) offsets = pid_block * BLOCK_H + tl.arange(0, BLOCK_H) mask = offsets < hidden_dim head = offsets // HEAD_DIM d = offsets - head * HEAD_DIM attn_off = pid_row * hidden_dim + offsets attn = tl.load(attn_output_ptr + attn_off, mask=mask, other=0.0).to(tl.float32) gate_off = pid_row * gate_stride_row + head * gate_stride_head + d g = tl.load(gate_ptr + gate_off, mask=mask, other=0.0).to(tl.float32) result = attn * tl.sigmoid(g) tl.store(output_ptr + attn_off, result, mask=mask) def fused_sigmoid_mul(attn_output: torch.Tensor, gate: torch.Tensor, inplace: bool=False) -> torch.Tensor: """ Fused sigmoid-mul for attention output gating. Equivalent to: attn_output * sigmoid(gate) The production Qwen3.5 path passes a 3D strided gate. A single hidden-block Triton kernel handles both that path and flat contiguous inputs. When inplace=True, writes result back to attn_output and returns it. Supports strided gate: if gate is 3D (num_tokens, num_heads, head_dim) and attn_output is 2D (num_tokens, hidden_dim), the kernel reads gate via explicit strides without requiring a contiguous copy. """ if gate.ndim == 3 and attn_output.ndim == 2: num_tokens, num_heads, head_dim = gate.shape hidden_dim = num_heads * head_dim assert attn_output.shape == (num_tokens, hidden_dim) gate_stride_row = gate.stride(0) gate_stride_head = gate.stride(1) else: assert attn_output.shape == gate.shape, 'attn_output and gate must have the same shape' hidden_dim = attn_output.shape[-1] num_tokens = attn_output.numel() // hidden_dim head_dim = hidden_dim gate_stride_row = hidden_dim gate_stride_head = hidden_dim out = attn_output if inplace else torch.empty_like(attn_output) block_h = 1024 if num_tokens < 1024 else 2048 grid = (num_tokens, triton.cdiv(hidden_dim, block_h)) _fused_sigmoid_mul_kernel[grid](out, attn_output, gate, gate_stride_row, gate_stride_head, hidden_dim, HEAD_DIM=head_dim, BLOCK_H=block_h, num_warps=4) return out @triton.jit def _silu_mul_kernel( gate_ptr: tl.tensor, up_ptr: tl.tensor, output_ptr: tl.tensor, elements: tl.constexpr, BLOCK: tl.constexpr, ) -> None: offsets = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK) mask = offsets < elements gate = tl.load(gate_ptr + offsets, mask=mask, other=0.0).to(tl.float32) up = tl.load(up_ptr + offsets, mask=mask, other=0.0).to(tl.float32) # Preserve the qualified CUDA --use_fast_math operations, including FTZ. output = tl.inline_asm_elementwise( "{ .reg .f32 exponent, denominator, activated; " "mul.ftz.f32 exponent, $1, 0fBFB8AA3B; " "ex2.approx.ftz.f32 exponent, exponent; " "add.ftz.f32 denominator, exponent, 0f3F800000; " "div.approx.ftz.f32 activated, $1, denominator; " "mul.ftz.f32 $0, $2, activated; }", constraints="=f,f,f", args=[gate, up], dtype=tl.float32, is_pure=True, pack=1, ) tl.store(output_ptr + offsets, output, mask=mask) def silu_mul(gate: torch.Tensor, up: torch.Tensor) -> torch.Tensor: """Match the qualified fused CUDA SiLU-mul before the BF16 output store.""" if (gate.shape != up.shape or gate.device != up.device or gate.dtype != torch.bfloat16 or up.dtype != torch.bfloat16 or not gate.is_cuda or not gate.is_contiguous() or not up.is_contiguous()): raise ValueError("Expected aligned contiguous CUDA BF16 gate and up tensors") output = torch.empty_like(gate) _silu_mul_kernel[(triton.cdiv(gate.numel(), 2048),)]( gate, up, output, gate.numel(), BLOCK=2048, num_warps=8, ) return output # MIT License # # Copyright 2025 Thinking Machines Lab # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # The above copyright notice and this permission notice shall be included in all # copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. def get_device_core_count() -> int: return torch.cuda.get_device_properties(torch.cuda.current_device()).multi_processor_count @triton.jit def _compute_pid(tile_id: tl.tensor, num_pid_in_group: tl.tensor, num_pid_m: tl.tensor, GROUP_SIZE_M: tl.tensor, NUM_SMS: tl.tensor) -> tuple[tl.tensor, tl.tensor]: group_id = tile_id // num_pid_in_group first_pid_m = group_id * GROUP_SIZE_M group_size_m = min(num_pid_m - first_pid_m, GROUP_SIZE_M) pid_m = first_pid_m + tile_id % group_size_m pid_n = tile_id % num_pid_in_group // group_size_m return (pid_m, pid_n) @triton.jit def matmul_kernel_persistent(a_ptr: tl.tensor, b_ptr: tl.tensor, c_ptr: tl.tensor, bias_ptr: tl.tensor, M: tl.int32, N: tl.int32, K: tl.int32, stride_am: tl.int32, stride_ak: tl.int32, stride_bk: tl.int32, stride_bn: tl.int32, stride_cm: tl.int32, stride_cn: tl.int32, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, NUM_SMS: tl.constexpr, A_LARGE: tl.constexpr, B_LARGE: tl.constexpr, C_LARGE: tl.constexpr, HAS_BIAS: tl.constexpr) -> None: start_pid = tl.program_id(axis=0) num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) k_tiles = tl.cdiv(K, BLOCK_SIZE_K) num_tiles = num_pid_m * num_pid_n offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K) num_pid_in_group = GROUP_SIZE_M * num_pid_n for tile_id in tl.range(start_pid, num_tiles, NUM_SMS, flatten=True): (pid_m, pid_n) = _compute_pid(tile_id, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS) start_m = pid_m * BLOCK_SIZE_M start_n = pid_n * BLOCK_SIZE_N offs_am = start_m + tl.arange(0, BLOCK_SIZE_M) offs_bn = start_n + tl.arange(0, BLOCK_SIZE_N) if A_LARGE: offs_am = offs_am.to(tl.int64) if B_LARGE: offs_bn = offs_bn.to(tl.int64) offs_am = tl.where(offs_am < M, offs_am, 0) offs_bn = tl.where(offs_bn < N, offs_bn, 0) offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M) offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N) accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) for ki in range(k_tiles): if A_LARGE or B_LARGE: offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K).to(tl.int64) else: offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) a_ptrs = a_ptr + (offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) b_ptrs = b_ptr + (offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) a = tl.load(a_ptrs, mask=offs_k_for_mask[None, :] < K - ki * BLOCK_SIZE_K, other=0.0) b = tl.load(b_ptrs, mask=offs_k_for_mask[:, None] < K - ki * BLOCK_SIZE_K, other=0.0) accumulator = tl.dot(a, b, accumulator) offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) if C_LARGE: offs_cm = offs_cm.to(tl.int64) offs_cn = offs_cn.to(tl.int64) c_ptrs = c_ptr + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) if HAS_BIAS: bias_ptrs = bias_ptr + offs_cn bias = tl.load(bias_ptrs, mask=offs_cn < N, other=0.0).to(tl.float32) accumulator += bias if c_ptr.dtype.element_ty == tl.float8e4nv: c = accumulator.to(tl.float8e4nv) elif c_ptr.dtype.element_ty == tl.bfloat16: c = accumulator.to(tl.bfloat16) elif c_ptr.dtype.element_ty == tl.float32: c = accumulator.to(tl.float32) else: c = accumulator.to(tl.float16) tl.store(c_ptrs, c, mask=c_mask) def _matmul_persistent_triton(a: torch.Tensor, b: torch.Tensor, bias: torch.Tensor | None=None) -> torch.Tensor: assert a.shape[1] == b.shape[0], 'Incompatible dimensions' assert a.dtype == b.dtype, 'Incompatible dtypes' assert bias is None or bias.dim() == 1, 'Expected a one-dimensional bias' NUM_SMS = get_device_core_count() (M, K) = a.shape (K, N) = b.shape dtype = a.dtype c = torch.empty((M, N), device=a.device, dtype=dtype) def grid(META: dict[str, Any]) -> tuple[int]: return (min(NUM_SMS, triton.cdiv(M, META['BLOCK_SIZE_M']) * triton.cdiv(N, META['BLOCK_SIZE_N'])),) configs = {torch.bfloat16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float32: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}} matmul_kernel_persistent[grid](a, b, c, bias, M, N, K, a.stride(0), a.stride(1), b.stride(0), b.stride(1), c.stride(0), c.stride(1), NUM_SMS=NUM_SMS, A_LARGE=a.numel() > 2 ** 31, B_LARGE=b.numel() > 2 ** 31, C_LARGE=c.numel() > 2 ** 31, HAS_BIAS=bias is not None, **configs[dtype]) return c @triton.jit def mean_kernel(input_ptr: tl.tensor, output_ptr: tl.tensor, input_stride0: tl.int32, input_stride1: tl.int32, input_stride2: tl.int32, output_stride0: tl.int32, output_stride1: tl.int32, M: tl.int32, N: tl.int32, K: tl.int32, BLOCK_SIZE: tl.constexpr) -> None: """ Kernel for computing mean along a single dimension. Input is viewed as (M, N, K) where N is the dimension being reduced. """ pid = tl.program_id(0) m_idx = pid // K k_idx = pid % K if m_idx >= M or k_idx >= K: return acc = 0.0 for n_start in range(0, N, BLOCK_SIZE): n_offsets = n_start + tl.arange(0, BLOCK_SIZE) mask = n_offsets < N input_idx = m_idx * input_stride0 + n_offsets * input_stride1 + k_idx * input_stride2 vals = tl.load(input_ptr + input_idx, mask=mask, other=0.0) acc += tl.sum(vals) mean_val = acc / N output_idx = m_idx * output_stride0 + k_idx * output_stride1 tl.store(output_ptr + output_idx, mean_val) def mean_dim(input: torch.Tensor, dim: int, keepdim: bool=False, dtype: torch.dtype | None=None) -> torch.Tensor: """ Triton implementation of torch.mean with single dimension reduction. Args: input: Input tensor dim: Single dimension along which to compute mean keepdim: Whether to keep the reduced dimension dtype: Output dtype. If None, uses input dtype (or float32 for integer inputs) Returns: Tensor with mean values along specified dimension """ assert input.is_cuda or input.is_xpu, 'Input must be a CUDA or XPU tensor' assert -input.ndim <= dim < input.ndim, f'Invalid dimension {dim} for tensor with {input.ndim} dimensions' if dim < 0: dim = dim + input.ndim if dtype is None: if input.dtype in [torch.int8, torch.int16, torch.int32, torch.int64]: dtype = torch.float32 else: dtype = input.dtype if input.dtype != dtype: input = input.to(dtype) shape = list(input.shape) M = 1 for i in range(dim): M *= shape[i] N = shape[dim] K = 1 for i in range(dim + 1, len(shape)): K *= shape[i] input_3d = input.reshape(M, N, K) if keepdim: output_shape = shape.copy() output_shape[dim] = 1 else: output_shape = shape[:dim] + shape[dim + 1:] output = torch.empty(output_shape, dtype=dtype, device=input.device) if keepdim: output_2d = output.reshape(M, 1, K).squeeze(1) else: output_2d = output.reshape(M, K) grid = (M * K,) BLOCK_SIZE = 1024 mean_kernel[grid](input_3d, output_2d, input_3d.stride(0), input_3d.stride(1), input_3d.stride(2), output_2d.stride(0), output_2d.stride(1) if output_2d.ndim > 1 else 0, M, N, K, BLOCK_SIZE) return output def mean_batch_invariant(input: torch.Tensor, dim: list[int], keepdim: bool=False, dtype: torch.dtype | None=None) -> torch.Tensor: assert dtype is None or dtype == torch.float32, f'unsupported dtype: {dtype}' if len(dim) == 1: return mean_dim(input, dim[0], keepdim=keepdim) else: assert input.dtype in {torch.float16, torch.bfloat16, torch.float32}, 'only float types supported for now' n_elems = 1 for d in dim: n_elems *= input.shape[d] return torch.sum(input, dim=dim, keepdim=keepdim, dtype=torch.float32) / n_elems @triton.jit def bmm_kernel_persistent(a_ptr: tl.tensor, b_ptr: tl.tensor, c_ptr: tl.tensor, B: tl.int32, M: tl.int32, N: tl.int32, K: tl.int32, stride_ab: tl.int32, stride_am: tl.int32, stride_ak: tl.int32, stride_bb: tl.int32, stride_bk: tl.int32, stride_bn: tl.int32, stride_cb: tl.int32, stride_cm: tl.int32, stride_cn: tl.int32, BLOCK_SIZE_M: tl.constexpr, BLOCK_SIZE_N: tl.constexpr, BLOCK_SIZE_K: tl.constexpr, GROUP_SIZE_M: tl.constexpr, NUM_SMS: tl.constexpr, A_LARGE: tl.constexpr, B_LARGE: tl.constexpr, C_LARGE: tl.constexpr) -> None: """ Batched matrix multiplication kernel that processes batches in parallel. Each tile processes a (BLOCK_SIZE_M, BLOCK_SIZE_N) output block for a specific batch. """ start_pid = tl.program_id(axis=0) num_pid_m = tl.cdiv(M, BLOCK_SIZE_M) num_pid_n = tl.cdiv(N, BLOCK_SIZE_N) k_tiles = tl.cdiv(K, BLOCK_SIZE_K) num_tiles_per_batch = num_pid_m * num_pid_n num_tiles_total = B * num_tiles_per_batch offs_k_for_mask = tl.arange(0, BLOCK_SIZE_K) num_pid_in_group = GROUP_SIZE_M * num_pid_n for tile_id in tl.range(start_pid, num_tiles_total, NUM_SMS, flatten=True): batch_idx = tile_id // num_tiles_per_batch tile_in_batch = tile_id % num_tiles_per_batch (pid_m, pid_n) = _compute_pid(tile_in_batch, num_pid_in_group, num_pid_m, GROUP_SIZE_M, NUM_SMS) start_m = pid_m * BLOCK_SIZE_M start_n = pid_n * BLOCK_SIZE_N offs_am = start_m + tl.arange(0, BLOCK_SIZE_M) offs_bn = start_n + tl.arange(0, BLOCK_SIZE_N) if A_LARGE: offs_am = offs_am.to(tl.int64) if B_LARGE: offs_bn = offs_bn.to(tl.int64) offs_am = tl.where(offs_am < M, offs_am, 0) offs_bn = tl.where(offs_bn < N, offs_bn, 0) offs_am = tl.max_contiguous(tl.multiple_of(offs_am, BLOCK_SIZE_M), BLOCK_SIZE_M) offs_bn = tl.max_contiguous(tl.multiple_of(offs_bn, BLOCK_SIZE_N), BLOCK_SIZE_N) if A_LARGE or B_LARGE: batch_idx_typed = batch_idx.to(tl.int64) else: batch_idx_typed = batch_idx accumulator = tl.zeros((BLOCK_SIZE_M, BLOCK_SIZE_N), dtype=tl.float32) for ki in range(k_tiles): if A_LARGE or B_LARGE: offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K).to(tl.int64) else: offs_k = ki * BLOCK_SIZE_K + tl.arange(0, BLOCK_SIZE_K) a_ptrs = a_ptr + (batch_idx_typed * stride_ab + offs_am[:, None] * stride_am + offs_k[None, :] * stride_ak) b_ptrs = b_ptr + (batch_idx_typed * stride_bb + offs_k[:, None] * stride_bk + offs_bn[None, :] * stride_bn) a = tl.load(a_ptrs, mask=offs_k_for_mask[None, :] < K - ki * BLOCK_SIZE_K, other=0.0) b = tl.load(b_ptrs, mask=offs_k_for_mask[:, None] < K - ki * BLOCK_SIZE_K, other=0.0) accumulator = tl.dot(a, b, accumulator) offs_cm = pid_m * BLOCK_SIZE_M + tl.arange(0, BLOCK_SIZE_M) offs_cn = pid_n * BLOCK_SIZE_N + tl.arange(0, BLOCK_SIZE_N) if C_LARGE: offs_cm = offs_cm.to(tl.int64) offs_cn = offs_cn.to(tl.int64) c_ptrs = c_ptr + batch_idx_typed * stride_cb + stride_cm * offs_cm[:, None] + stride_cn * offs_cn[None, :] c_mask = (offs_cm[:, None] < M) & (offs_cn[None, :] < N) if c_ptr.dtype.element_ty == tl.float8e4nv: c = accumulator.to(tl.float8e4nv) elif c_ptr.dtype.element_ty == tl.bfloat16: c = accumulator.to(tl.bfloat16) elif c_ptr.dtype.element_ty == tl.float32: c = accumulator.to(tl.float32) else: c = accumulator.to(tl.float16) tl.store(c_ptrs, c, mask=c_mask) def bmm_batch_invariant(a: torch.Tensor, b: torch.Tensor, *, out: torch.Tensor | None=None) -> torch.Tensor: if a.ndim == 3 and b.ndim == 3: assert a.shape[0] == b.shape[0], 'Batch sizes must match' assert a.shape[2] == b.shape[1], 'Incompatible dimensions' assert a.dtype == b.dtype, 'Incompatible dtypes' B = a.shape[0] M = a.shape[1] K = a.shape[2] N = b.shape[2] dtype = a.dtype if out is None: c = torch.empty((B, M, N), device=a.device, dtype=dtype) else: c = out NUM_SMS = get_device_core_count() configs = {torch.bfloat16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float16: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 256, 'BLOCK_SIZE_K': 64, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}, torch.float32: {'BLOCK_SIZE_M': 128, 'BLOCK_SIZE_N': 128, 'BLOCK_SIZE_K': 32, 'GROUP_SIZE_M': 8, 'num_stages': 3, 'num_warps': 8}} config = configs.get(dtype) if config is None: raise ValueError(f'Unsupported dtype {dtype} for bmm_batch_invariant. Supported dtypes are: {list(configs.keys())}') num_tiles_per_batch = triton.cdiv(M, config['BLOCK_SIZE_M']) * triton.cdiv(N, config['BLOCK_SIZE_N']) num_tiles_total = B * num_tiles_per_batch grid = (min(NUM_SMS, num_tiles_total),) bmm_kernel_persistent[grid](a, b, c, B, M, N, K, a.stride(0), a.stride(1), a.stride(2), b.stride(0), b.stride(1), b.stride(2), c.stride(0), c.stride(1), c.stride(2), NUM_SMS=NUM_SMS, A_LARGE=a.numel() > 2 ** 31, B_LARGE=b.numel() > 2 ** 31, C_LARGE=c.numel() > 2 ** 31, **config) return c else: raise ValueError(f'bmm_batch_invariant expects 3D tensors, got shapes {a.shape} and {b.shape}') @triton.jit(do_not_specialize=["count"]) def _bf16_pair_kernel(value: tl.tensor, high: tl.tensor, low: tl.tensor, count: tl.int32, limit: tl.constexpr, block: tl.constexpr) -> None: offsets = tl.program_id(0) * block + tl.arange(0, block) current = tl.load(value + offsets, offsets < count, other=0) clipped = tl.minimum(tl.maximum(current, -limit, propagate_nan=tl.PropagateNan.ALL), limit, propagate_nan=tl.PropagateNan.ALL) upper = clipped.to(tl.bfloat16) lower = (current - upper.to(tl.float32)).to(tl.bfloat16) lower = tl.where(current == 0, upper, lower) tl.store(high + offsets, upper, offsets < count) tl.store(low + offsets, lower, offsets < count) def bf16_pair_sum(value: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]: if not value.is_cuda or value.dtype != torch.float32 or not value.is_contiguous(): raise ValueError("Expected contiguous CUDA FP32 residuals") high = torch.empty_like(value, dtype=torch.bfloat16) low = torch.empty_like(value, dtype=torch.bfloat16) count = value.numel() if count: _bf16_pair_kernel[(triton.cdiv(count, 1024),)]( value, high, low, count, torch.finfo(torch.bfloat16).max, 1024, num_warps=4, enable_fp_fusion=False, ) return high, low @triton.jit(do_not_specialize=["count"]) def _residual_kernel(first: tl.tensor, second: tl.tensor, output: tl.tensor, count: tl.int32, block: tl.constexpr) -> None: offsets = tl.program_id(0) * block + tl.arange(0, block) upper = tl.load(first + offsets, offsets < count, other=0).to(tl.bfloat16).to(tl.float32) lower = tl.load(second + offsets, offsets < count, other=0).to(tl.float32) tl.store(output + offsets, upper + lower, offsets < count) def residual_sum(first: torch.Tensor, second: torch.Tensor) -> torch.Tensor: if (not first.is_cuda or second.device != first.device or second.shape != first.shape or first.dtype not in (torch.bfloat16, torch.float32) or second.dtype != torch.bfloat16 or not first.is_contiguous() or not second.is_contiguous()): raise ValueError("Expected aligned contiguous CUDA residuals") output = torch.empty_like(first, dtype=torch.float32) count = first.numel() if count: _residual_kernel[(triton.cdiv(count, 1024),)]( first, second, output, count, 1024, num_warps=4, enable_fp_fusion=False, ) return output