WarpStateAI / ops /matmul_kernels.py
zerdovzad's picture
Upload 139 files
c8ba9c6 verified
Raw History Blame Contribute Delete
9.78 kB
"""Bounded BF16 matrix kernels; autograd and policy selection belong to callers.
Triton variants accumulate dot products in FP32 and return BF16, without split-K
atomics or reduced-precision operands. ``aten`` preserves the process's existing
BLAS settings. Persistent scheduling assigns multiple output tiles to each
resident program; ``persistent_tma`` also uses TensorDescriptor loads. These
kernels do not use architecture-specific warp-specialized execution.
"""
from collections.abc import Mapping
import json
import torch
from torch.library import wrap_triton
import triton
import triton.language as tl
ALGORITHMS = ("aten", "tiled", "persistent", "persistent_tma")
TILE_FIELDS = ("block_m", "block_n", "block_k", "num_warps", "num_stages")
DEFAULT_TILE = (128, 128, 32, 4, 3)
TUNING_TILES = (
(64, 128, 64, 4, 3),
DEFAULT_TILE,
(128, 256, 32, 8, 3),
(256, 128, 32, 8, 3),
(128, 128, 64, 4, 2),
)
def normalize_tile(tile=None):
"""Normalize a five-integer tile tuple or named JSON object."""
if tile is None:
tile = DEFAULT_TILE
if isinstance(tile, Mapping):
if set(tile) != set(TILE_FIELDS):
raise ValueError("Tile must contain exactly " + ", ".join(TILE_FIELDS))
tile = tuple(tile[key] for key in TILE_FIELDS)
if not isinstance(tile, (tuple, list)) or len(tile) != 5:
raise ValueError("Tile must be (block_m, block_n, block_k, num_warps, num_stages)")
if any(type(value) is not int for value in tile):
raise ValueError("Tile values must be integers")
bm, bn, bk, warps, stages = tile
if any(value < 16 or value > 256 or value & (value - 1) for value in (bm, bn, bk)):
raise ValueError("Matrix tile dimensions must be powers of two between 16 and 256")
if warps not in (4, 8) or stages not in (1, 2, 3, 4):
raise ValueError("Supported launch settings are 4/8 warps and 1–4 stages")
if (bm + bn) * bk * 2 * stages > 96 * 1024:
raise ValueError("BF16 operand pipeline exceeds the conservative 96 KiB shared-memory budget")
return (bm, bn, bk, warps, stages)
def tile_dict(tile):
return dict(zip(TILE_FIELDS, normalize_tile(tile), strict=True))
def matrix_signature(a, b):
"""Serializable dispatch signature; use outside captured execution."""
if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0]:
raise ValueError("Expected compatible two-dimensional matrix operands")
if a.dtype != b.dtype:
raise ValueError("Matrix operands must have matching dtypes")
return {"m": a.shape[0], "n": b.shape[1], "k": a.shape[1],
"a_stride": list(a.stride()), "b_stride": list(b.stride()),
"dtype": str(a.dtype).removeprefix("torch.")}
def policy_key(a, b):
return json.dumps(matrix_signature(a, b), sort_keys=True, separators=(",", ":"))
@triton.jit
def _tile_position(tile_id, tiles_m, tiles_n, GROUP_M: tl.constexpr):
group_size = GROUP_M * tiles_n
first_m = (tile_id // group_size) * GROUP_M
actual_m = tl.minimum(tiles_m - first_m, GROUP_M)
within = tile_id % group_size
return first_m + within % actual_m, within // actual_m
@triton.jit
def _matmul_kernel(A, B, C, M: tl.constexpr, N: tl.constexpr, K: tl.constexpr,
A0: tl.constexpr, A1: tl.constexpr, B0: tl.constexpr, B1: tl.constexpr,
BM: tl.constexpr, BN: tl.constexpr, BK: tl.constexpr,
GROUP_M: tl.constexpr, PROGRAMS: tl.constexpr,
PERSISTENT: tl.constexpr):
tiles_m = tl.cdiv(M, BM)
tiles_n = tl.cdiv(N, BN)
first = tl.program_id(0)
# A tiled launch has one iteration. Persistent programs advance through
# multiple output tiles, keeping the launch bounded by the SM count.
stop = tiles_m * tiles_n if PERSISTENT else first + 1
step = PROGRAMS if PERSISTENT else 1
for tile_id in range(first, stop, step):
tile_m, tile_n = _tile_position(tile_id, tiles_m, tiles_n, GROUP_M)
rows = tile_m * BM + tl.arange(0, BM)
cols = tile_n * BN + tl.arange(0, BN)
reduction = tl.arange(0, BK)
accumulator = tl.full((BM, BN), 0, tl.float32)
for block in range(tl.cdiv(K, BK)):
kk = block * BK + reduction
av = tl.load(A + rows[:, None] * A0 + kk[None, :] * A1,
(rows[:, None] < M) & (kk[None, :] < K), 0)
bv = tl.load(B + kk[:, None] * B0 + cols[None, :] * B1,
(kk[:, None] < K) & (cols[None, :] < N), 0)
accumulator = tl.dot(av, bv, accumulator, input_precision="ieee")
tl.store(C + rows[:, None] * N + cols[None, :], accumulator.to(tl.bfloat16),
(rows[:, None] < M) & (cols[None, :] < N))
@triton.jit
def _persistent_tma_kernel(A_DESC, B_DESC, C,
M: tl.constexpr, N: tl.constexpr, K: tl.constexpr,
BM: tl.constexpr, BN: tl.constexpr, BK: tl.constexpr,
PROGRAMS: tl.constexpr, A_ROW: tl.constexpr, B_ROW: tl.constexpr):
tiles_m = tl.cdiv(M, BM)
tiles_n = tl.cdiv(N, BN)
for tile_id in tl.range(tl.program_id(0), tiles_m * tiles_n, PROGRAMS):
tile_m, tile_n = _tile_position(tile_id, tiles_m, tiles_n, 8)
row = (tile_m * BM).to(tl.int32)
col = (tile_n * BN).to(tl.int32)
accumulator = tl.full((BM, BN), 0, tl.float32)
for block in tl.range(tl.cdiv(K, BK)):
kk = (block * BK).to(tl.int32)
av = A_DESC.load([row, kk] if A_ROW else [kk, row])
bv = B_DESC.load([kk, col] if B_ROW else [col, kk])
accumulator = tl.dot(av if A_ROW else av.T, bv if B_ROW else bv.T,
accumulator, input_precision="ieee")
rows = row + tl.arange(0, BM)
cols = col + tl.arange(0, BN)
tl.store(C + rows[:, None] * N + cols[None, :], accumulator.to(tl.bfloat16),
(rows[:, None] < M) & (cols[None, :] < N))
@torch.library.custom_op("warpstate::persistent_tma_mm", mutates_args=())
def _tma_mm(a: torch.Tensor, b: torch.Tensor, bm: int, bn: int, bk: int,
warps: int, stages: int) -> torch.Tensor:
"""Opaque launch boundary for the host descriptor API in PyTorch 2.14.
That release's triton_op/AOT path does not preserve host descriptor metadata.
This custom-op boundary keeps the TMA implementation compilable without
substituting a different algorithm or making inputs appear mutable.
"""
from triton.tools.tensor_descriptor import TensorDescriptor
m, k = a.shape
n = b.shape[1]
a_row, b_row = a.stride(1) == 1, b.stride(1) == 1
a_desc = TensorDescriptor.from_tensor(a if a_row else a.T,
[bm, bk] if a_row else [bk, bm])
b_desc = TensorDescriptor.from_tensor(b if b_row else b.T,
[bk, bn] if b_row else [bn, bk])
output = torch.empty((m, n), device=a.device, dtype=torch.bfloat16)
programs = min(triton.cdiv(m, bm) * triton.cdiv(n, bn),
torch.cuda.get_device_properties(a.device).multi_processor_count)
_persistent_tma_kernel[(programs,)](
a_desc, b_desc, output, m, n, k, bm, bn, bk, programs, a_row, b_row,
num_warps=warps, num_stages=stages)
return output
@_tma_mm.register_fake
def _tma_mm_fake(a, b, bm, bn, bk, warps, stages):
return torch.empty((a.shape[0], b.shape[1]), device=a.device, dtype=torch.bfloat16)
def launch_mm(a, b, algorithm="aten", tile=None):
"""Return CUDA BF16 ``a @ b``; caller supplies any autograd registration.
Supports positive 2D strides, including ordinary transposed weights and
transposed output gradients. No inputs or parameters are modified. The
conservative tile footprint check is not a guarantee of compiler resource
usage; compilation failures remain explicit and are recorded by the tuner.
"""
if algorithm not in ALGORITHMS:
raise ValueError("Unknown matrix algorithm: " + str(algorithm))
if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0]:
raise ValueError("Expected compatible two-dimensional matrix operands")
if a.device != b.device or not a.is_cuda:
raise ValueError("Matrix kernels require operands on the same CUDA device")
if a.dtype != torch.bfloat16 or b.dtype != torch.bfloat16:
raise ValueError("Matrix kernels require BF16 inputs")
if min(a.shape[0], a.shape[1], b.shape[1]) < 1 or min(*a.stride(), *b.stride()) < 1:
raise ValueError("Matrix kernels require positive dimensions and strides")
if algorithm == "aten":
return torch.mm(a, b)
bm, bn, bk, warps, stages = normalize_tile(tile)
m, k = a.shape
n = b.shape[1]
if algorithm == "persistent_tma":
if (min(a.stride()) != 1 or min(b.stride()) != 1
or max(a.stride()) * 2 % 16 or max(b.stride()) * 2 % 16):
raise ValueError("TMA operands require an innermost stride of one and 16-byte-aligned rows")
if a.storage_offset() * 2 % 16 or b.storage_offset() * 2 % 16:
raise ValueError("TMA operands require 16-byte-aligned storage offsets")
return _tma_mm(a, b, bm, bn, bk, warps, stages)
output = torch.empty((m, n), device=a.device, dtype=torch.bfloat16)
tiles = triton.cdiv(m, bm) * triton.cdiv(n, bn)
programs = min(tiles, torch.cuda.get_device_properties(a.device).multi_processor_count) \
if algorithm == "persistent" else tiles
wrap_triton(_matmul_kernel)[(programs,)](
a, b, output, m, n, k, a.stride(0), a.stride(1), b.stride(0), b.stride(1),
bm, bn, bk, 8, programs, algorithm == "persistent",
num_warps=warps, num_stages=stages)
return output