File size: 9,781 Bytes
c8ba9c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 | """Bounded BF16 matrix kernels; autograd and policy selection belong to callers.
Triton variants accumulate dot products in FP32 and return BF16, without split-K
atomics or reduced-precision operands. ``aten`` preserves the process's existing
BLAS settings. Persistent scheduling assigns multiple output tiles to each
resident program; ``persistent_tma`` also uses TensorDescriptor loads. These
kernels do not use architecture-specific warp-specialized execution.
"""
from collections.abc import Mapping
import json
import torch
from torch.library import wrap_triton
import triton
import triton.language as tl
ALGORITHMS = ("aten", "tiled", "persistent", "persistent_tma")
TILE_FIELDS = ("block_m", "block_n", "block_k", "num_warps", "num_stages")
DEFAULT_TILE = (128, 128, 32, 4, 3)
TUNING_TILES = (
(64, 128, 64, 4, 3),
DEFAULT_TILE,
(128, 256, 32, 8, 3),
(256, 128, 32, 8, 3),
(128, 128, 64, 4, 2),
)
def normalize_tile(tile=None):
"""Normalize a five-integer tile tuple or named JSON object."""
if tile is None:
tile = DEFAULT_TILE
if isinstance(tile, Mapping):
if set(tile) != set(TILE_FIELDS):
raise ValueError("Tile must contain exactly " + ", ".join(TILE_FIELDS))
tile = tuple(tile[key] for key in TILE_FIELDS)
if not isinstance(tile, (tuple, list)) or len(tile) != 5:
raise ValueError("Tile must be (block_m, block_n, block_k, num_warps, num_stages)")
if any(type(value) is not int for value in tile):
raise ValueError("Tile values must be integers")
bm, bn, bk, warps, stages = tile
if any(value < 16 or value > 256 or value & (value - 1) for value in (bm, bn, bk)):
raise ValueError("Matrix tile dimensions must be powers of two between 16 and 256")
if warps not in (4, 8) or stages not in (1, 2, 3, 4):
raise ValueError("Supported launch settings are 4/8 warps and 1–4 stages")
if (bm + bn) * bk * 2 * stages > 96 * 1024:
raise ValueError("BF16 operand pipeline exceeds the conservative 96 KiB shared-memory budget")
return (bm, bn, bk, warps, stages)
def tile_dict(tile):
return dict(zip(TILE_FIELDS, normalize_tile(tile), strict=True))
def matrix_signature(a, b):
"""Serializable dispatch signature; use outside captured execution."""
if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0]:
raise ValueError("Expected compatible two-dimensional matrix operands")
if a.dtype != b.dtype:
raise ValueError("Matrix operands must have matching dtypes")
return {"m": a.shape[0], "n": b.shape[1], "k": a.shape[1],
"a_stride": list(a.stride()), "b_stride": list(b.stride()),
"dtype": str(a.dtype).removeprefix("torch.")}
def policy_key(a, b):
return json.dumps(matrix_signature(a, b), sort_keys=True, separators=(",", ":"))
@triton.jit
def _tile_position(tile_id, tiles_m, tiles_n, GROUP_M: tl.constexpr):
group_size = GROUP_M * tiles_n
first_m = (tile_id // group_size) * GROUP_M
actual_m = tl.minimum(tiles_m - first_m, GROUP_M)
within = tile_id % group_size
return first_m + within % actual_m, within // actual_m
@triton.jit
def _matmul_kernel(A, B, C, M: tl.constexpr, N: tl.constexpr, K: tl.constexpr,
A0: tl.constexpr, A1: tl.constexpr, B0: tl.constexpr, B1: tl.constexpr,
BM: tl.constexpr, BN: tl.constexpr, BK: tl.constexpr,
GROUP_M: tl.constexpr, PROGRAMS: tl.constexpr,
PERSISTENT: tl.constexpr):
tiles_m = tl.cdiv(M, BM)
tiles_n = tl.cdiv(N, BN)
first = tl.program_id(0)
# A tiled launch has one iteration. Persistent programs advance through
# multiple output tiles, keeping the launch bounded by the SM count.
stop = tiles_m * tiles_n if PERSISTENT else first + 1
step = PROGRAMS if PERSISTENT else 1
for tile_id in range(first, stop, step):
tile_m, tile_n = _tile_position(tile_id, tiles_m, tiles_n, GROUP_M)
rows = tile_m * BM + tl.arange(0, BM)
cols = tile_n * BN + tl.arange(0, BN)
reduction = tl.arange(0, BK)
accumulator = tl.full((BM, BN), 0, tl.float32)
for block in range(tl.cdiv(K, BK)):
kk = block * BK + reduction
av = tl.load(A + rows[:, None] * A0 + kk[None, :] * A1,
(rows[:, None] < M) & (kk[None, :] < K), 0)
bv = tl.load(B + kk[:, None] * B0 + cols[None, :] * B1,
(kk[:, None] < K) & (cols[None, :] < N), 0)
accumulator = tl.dot(av, bv, accumulator, input_precision="ieee")
tl.store(C + rows[:, None] * N + cols[None, :], accumulator.to(tl.bfloat16),
(rows[:, None] < M) & (cols[None, :] < N))
@triton.jit
def _persistent_tma_kernel(A_DESC, B_DESC, C,
M: tl.constexpr, N: tl.constexpr, K: tl.constexpr,
BM: tl.constexpr, BN: tl.constexpr, BK: tl.constexpr,
PROGRAMS: tl.constexpr, A_ROW: tl.constexpr, B_ROW: tl.constexpr):
tiles_m = tl.cdiv(M, BM)
tiles_n = tl.cdiv(N, BN)
for tile_id in tl.range(tl.program_id(0), tiles_m * tiles_n, PROGRAMS):
tile_m, tile_n = _tile_position(tile_id, tiles_m, tiles_n, 8)
row = (tile_m * BM).to(tl.int32)
col = (tile_n * BN).to(tl.int32)
accumulator = tl.full((BM, BN), 0, tl.float32)
for block in tl.range(tl.cdiv(K, BK)):
kk = (block * BK).to(tl.int32)
av = A_DESC.load([row, kk] if A_ROW else [kk, row])
bv = B_DESC.load([kk, col] if B_ROW else [col, kk])
accumulator = tl.dot(av if A_ROW else av.T, bv if B_ROW else bv.T,
accumulator, input_precision="ieee")
rows = row + tl.arange(0, BM)
cols = col + tl.arange(0, BN)
tl.store(C + rows[:, None] * N + cols[None, :], accumulator.to(tl.bfloat16),
(rows[:, None] < M) & (cols[None, :] < N))
@torch.library.custom_op("warpstate::persistent_tma_mm", mutates_args=())
def _tma_mm(a: torch.Tensor, b: torch.Tensor, bm: int, bn: int, bk: int,
warps: int, stages: int) -> torch.Tensor:
"""Opaque launch boundary for the host descriptor API in PyTorch 2.14.
That release's triton_op/AOT path does not preserve host descriptor metadata.
This custom-op boundary keeps the TMA implementation compilable without
substituting a different algorithm or making inputs appear mutable.
"""
from triton.tools.tensor_descriptor import TensorDescriptor
m, k = a.shape
n = b.shape[1]
a_row, b_row = a.stride(1) == 1, b.stride(1) == 1
a_desc = TensorDescriptor.from_tensor(a if a_row else a.T,
[bm, bk] if a_row else [bk, bm])
b_desc = TensorDescriptor.from_tensor(b if b_row else b.T,
[bk, bn] if b_row else [bn, bk])
output = torch.empty((m, n), device=a.device, dtype=torch.bfloat16)
programs = min(triton.cdiv(m, bm) * triton.cdiv(n, bn),
torch.cuda.get_device_properties(a.device).multi_processor_count)
_persistent_tma_kernel[(programs,)](
a_desc, b_desc, output, m, n, k, bm, bn, bk, programs, a_row, b_row,
num_warps=warps, num_stages=stages)
return output
@_tma_mm.register_fake
def _tma_mm_fake(a, b, bm, bn, bk, warps, stages):
return torch.empty((a.shape[0], b.shape[1]), device=a.device, dtype=torch.bfloat16)
def launch_mm(a, b, algorithm="aten", tile=None):
"""Return CUDA BF16 ``a @ b``; caller supplies any autograd registration.
Supports positive 2D strides, including ordinary transposed weights and
transposed output gradients. No inputs or parameters are modified. The
conservative tile footprint check is not a guarantee of compiler resource
usage; compilation failures remain explicit and are recorded by the tuner.
"""
if algorithm not in ALGORITHMS:
raise ValueError("Unknown matrix algorithm: " + str(algorithm))
if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0]:
raise ValueError("Expected compatible two-dimensional matrix operands")
if a.device != b.device or not a.is_cuda:
raise ValueError("Matrix kernels require operands on the same CUDA device")
if a.dtype != torch.bfloat16 or b.dtype != torch.bfloat16:
raise ValueError("Matrix kernels require BF16 inputs")
if min(a.shape[0], a.shape[1], b.shape[1]) < 1 or min(*a.stride(), *b.stride()) < 1:
raise ValueError("Matrix kernels require positive dimensions and strides")
if algorithm == "aten":
return torch.mm(a, b)
bm, bn, bk, warps, stages = normalize_tile(tile)
m, k = a.shape
n = b.shape[1]
if algorithm == "persistent_tma":
if (min(a.stride()) != 1 or min(b.stride()) != 1
or max(a.stride()) * 2 % 16 or max(b.stride()) * 2 % 16):
raise ValueError("TMA operands require an innermost stride of one and 16-byte-aligned rows")
if a.storage_offset() * 2 % 16 or b.storage_offset() * 2 % 16:
raise ValueError("TMA operands require 16-byte-aligned storage offsets")
return _tma_mm(a, b, bm, bn, bk, warps, stages)
output = torch.empty((m, n), device=a.device, dtype=torch.bfloat16)
tiles = triton.cdiv(m, bm) * triton.cdiv(n, bn)
programs = min(tiles, torch.cuda.get_device_properties(a.device).multi_processor_count) \
if algorithm == "persistent" else tiles
wrap_triton(_matmul_kernel)[(programs,)](
a, b, output, m, n, k, a.stride(0), a.stride(1), b.stride(0), b.stride(1),
bm, bn, bk, 8, programs, algorithm == "persistent",
num_warps=warps, num_stages=stages)
return output
|