Download ops/matmul_kernels.py from zerdovzad/WarpStateAI: direct link, hf CLI and curl.
- Browser
- Download file 9.78 kB
-
https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/ops/matmul_kernels.py
- Command line
-
hf download hf://zerdovzad/WarpStateAI/ops/matmul_kernels.py
-
curl -L -o matmul_kernels.py https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/ops/matmul_kernels.py
9.78 kB
| """Bounded BF16 matrix kernels; autograd and policy selection belong to callers. | |
| Triton variants accumulate dot products in FP32 and return BF16, without split-K | |
| atomics or reduced-precision operands. ``aten`` preserves the process's existing | |
| BLAS settings. Persistent scheduling assigns multiple output tiles to each | |
| resident program; ``persistent_tma`` also uses TensorDescriptor loads. These | |
| kernels do not use architecture-specific warp-specialized execution. | |
| """ | |
| from collections.abc import Mapping | |
| import json | |
| import torch | |
| from torch.library import wrap_triton | |
| import triton | |
| import triton.language as tl | |
| ALGORITHMS = ("aten", "tiled", "persistent", "persistent_tma") | |
| TILE_FIELDS = ("block_m", "block_n", "block_k", "num_warps", "num_stages") | |
| DEFAULT_TILE = (128, 128, 32, 4, 3) | |
| TUNING_TILES = ( | |
| (64, 128, 64, 4, 3), | |
| DEFAULT_TILE, | |
| (128, 256, 32, 8, 3), | |
| (256, 128, 32, 8, 3), | |
| (128, 128, 64, 4, 2), | |
| ) | |
| def normalize_tile(tile=None): | |
| """Normalize a five-integer tile tuple or named JSON object.""" | |
| if tile is None: | |
| tile = DEFAULT_TILE | |
| if isinstance(tile, Mapping): | |
| if set(tile) != set(TILE_FIELDS): | |
| raise ValueError("Tile must contain exactly " + ", ".join(TILE_FIELDS)) | |
| tile = tuple(tile[key] for key in TILE_FIELDS) | |
| if not isinstance(tile, (tuple, list)) or len(tile) != 5: | |
| raise ValueError("Tile must be (block_m, block_n, block_k, num_warps, num_stages)") | |
| if any(type(value) is not int for value in tile): | |
| raise ValueError("Tile values must be integers") | |
| bm, bn, bk, warps, stages = tile | |
| if any(value < 16 or value > 256 or value & (value - 1) for value in (bm, bn, bk)): | |
| raise ValueError("Matrix tile dimensions must be powers of two between 16 and 256") | |
| if warps not in (4, 8) or stages not in (1, 2, 3, 4): | |
| raise ValueError("Supported launch settings are 4/8 warps and 1–4 stages") | |
| if (bm + bn) * bk * 2 * stages > 96 * 1024: | |
| raise ValueError("BF16 operand pipeline exceeds the conservative 96 KiB shared-memory budget") | |
| return (bm, bn, bk, warps, stages) | |
| def tile_dict(tile): | |
| return dict(zip(TILE_FIELDS, normalize_tile(tile), strict=True)) | |
| def matrix_signature(a, b): | |
| """Serializable dispatch signature; use outside captured execution.""" | |
| if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0]: | |
| raise ValueError("Expected compatible two-dimensional matrix operands") | |
| if a.dtype != b.dtype: | |
| raise ValueError("Matrix operands must have matching dtypes") | |
| return {"m": a.shape[0], "n": b.shape[1], "k": a.shape[1], | |
| "a_stride": list(a.stride()), "b_stride": list(b.stride()), | |
| "dtype": str(a.dtype).removeprefix("torch.")} | |
| def policy_key(a, b): | |
| return json.dumps(matrix_signature(a, b), sort_keys=True, separators=(",", ":")) | |
| def _tile_position(tile_id, tiles_m, tiles_n, GROUP_M: tl.constexpr): | |
| group_size = GROUP_M * tiles_n | |
| first_m = (tile_id // group_size) * GROUP_M | |
| actual_m = tl.minimum(tiles_m - first_m, GROUP_M) | |
| within = tile_id % group_size | |
| return first_m + within % actual_m, within // actual_m | |
| def _matmul_kernel(A, B, C, M: tl.constexpr, N: tl.constexpr, K: tl.constexpr, | |
| A0: tl.constexpr, A1: tl.constexpr, B0: tl.constexpr, B1: tl.constexpr, | |
| BM: tl.constexpr, BN: tl.constexpr, BK: tl.constexpr, | |
| GROUP_M: tl.constexpr, PROGRAMS: tl.constexpr, | |
| PERSISTENT: tl.constexpr): | |
| tiles_m = tl.cdiv(M, BM) | |
| tiles_n = tl.cdiv(N, BN) | |
| first = tl.program_id(0) | |
| # A tiled launch has one iteration. Persistent programs advance through | |
| # multiple output tiles, keeping the launch bounded by the SM count. | |
| stop = tiles_m * tiles_n if PERSISTENT else first + 1 | |
| step = PROGRAMS if PERSISTENT else 1 | |
| for tile_id in range(first, stop, step): | |
| tile_m, tile_n = _tile_position(tile_id, tiles_m, tiles_n, GROUP_M) | |
| rows = tile_m * BM + tl.arange(0, BM) | |
| cols = tile_n * BN + tl.arange(0, BN) | |
| reduction = tl.arange(0, BK) | |
| accumulator = tl.full((BM, BN), 0, tl.float32) | |
| for block in range(tl.cdiv(K, BK)): | |
| kk = block * BK + reduction | |
| av = tl.load(A + rows[:, None] * A0 + kk[None, :] * A1, | |
| (rows[:, None] < M) & (kk[None, :] < K), 0) | |
| bv = tl.load(B + kk[:, None] * B0 + cols[None, :] * B1, | |
| (kk[:, None] < K) & (cols[None, :] < N), 0) | |
| accumulator = tl.dot(av, bv, accumulator, input_precision="ieee") | |
| tl.store(C + rows[:, None] * N + cols[None, :], accumulator.to(tl.bfloat16), | |
| (rows[:, None] < M) & (cols[None, :] < N)) | |
| def _persistent_tma_kernel(A_DESC, B_DESC, C, | |
| M: tl.constexpr, N: tl.constexpr, K: tl.constexpr, | |
| BM: tl.constexpr, BN: tl.constexpr, BK: tl.constexpr, | |
| PROGRAMS: tl.constexpr, A_ROW: tl.constexpr, B_ROW: tl.constexpr): | |
| tiles_m = tl.cdiv(M, BM) | |
| tiles_n = tl.cdiv(N, BN) | |
| for tile_id in tl.range(tl.program_id(0), tiles_m * tiles_n, PROGRAMS): | |
| tile_m, tile_n = _tile_position(tile_id, tiles_m, tiles_n, 8) | |
| row = (tile_m * BM).to(tl.int32) | |
| col = (tile_n * BN).to(tl.int32) | |
| accumulator = tl.full((BM, BN), 0, tl.float32) | |
| for block in tl.range(tl.cdiv(K, BK)): | |
| kk = (block * BK).to(tl.int32) | |
| av = A_DESC.load([row, kk] if A_ROW else [kk, row]) | |
| bv = B_DESC.load([kk, col] if B_ROW else [col, kk]) | |
| accumulator = tl.dot(av if A_ROW else av.T, bv if B_ROW else bv.T, | |
| accumulator, input_precision="ieee") | |
| rows = row + tl.arange(0, BM) | |
| cols = col + tl.arange(0, BN) | |
| tl.store(C + rows[:, None] * N + cols[None, :], accumulator.to(tl.bfloat16), | |
| (rows[:, None] < M) & (cols[None, :] < N)) | |
| def _tma_mm(a: torch.Tensor, b: torch.Tensor, bm: int, bn: int, bk: int, | |
| warps: int, stages: int) -> torch.Tensor: | |
| """Opaque launch boundary for the host descriptor API in PyTorch 2.14. | |
| That release's triton_op/AOT path does not preserve host descriptor metadata. | |
| This custom-op boundary keeps the TMA implementation compilable without | |
| substituting a different algorithm or making inputs appear mutable. | |
| """ | |
| from triton.tools.tensor_descriptor import TensorDescriptor | |
| m, k = a.shape | |
| n = b.shape[1] | |
| a_row, b_row = a.stride(1) == 1, b.stride(1) == 1 | |
| a_desc = TensorDescriptor.from_tensor(a if a_row else a.T, | |
| [bm, bk] if a_row else [bk, bm]) | |
| b_desc = TensorDescriptor.from_tensor(b if b_row else b.T, | |
| [bk, bn] if b_row else [bn, bk]) | |
| output = torch.empty((m, n), device=a.device, dtype=torch.bfloat16) | |
| programs = min(triton.cdiv(m, bm) * triton.cdiv(n, bn), | |
| torch.cuda.get_device_properties(a.device).multi_processor_count) | |
| _persistent_tma_kernel[(programs,)]( | |
| a_desc, b_desc, output, m, n, k, bm, bn, bk, programs, a_row, b_row, | |
| num_warps=warps, num_stages=stages) | |
| return output | |
| def _tma_mm_fake(a, b, bm, bn, bk, warps, stages): | |
| return torch.empty((a.shape[0], b.shape[1]), device=a.device, dtype=torch.bfloat16) | |
| def launch_mm(a, b, algorithm="aten", tile=None): | |
| """Return CUDA BF16 ``a @ b``; caller supplies any autograd registration. | |
| Supports positive 2D strides, including ordinary transposed weights and | |
| transposed output gradients. No inputs or parameters are modified. The | |
| conservative tile footprint check is not a guarantee of compiler resource | |
| usage; compilation failures remain explicit and are recorded by the tuner. | |
| """ | |
| if algorithm not in ALGORITHMS: | |
| raise ValueError("Unknown matrix algorithm: " + str(algorithm)) | |
| if a.ndim != 2 or b.ndim != 2 or a.shape[1] != b.shape[0]: | |
| raise ValueError("Expected compatible two-dimensional matrix operands") | |
| if a.device != b.device or not a.is_cuda: | |
| raise ValueError("Matrix kernels require operands on the same CUDA device") | |
| if a.dtype != torch.bfloat16 or b.dtype != torch.bfloat16: | |
| raise ValueError("Matrix kernels require BF16 inputs") | |
| if min(a.shape[0], a.shape[1], b.shape[1]) < 1 or min(*a.stride(), *b.stride()) < 1: | |
| raise ValueError("Matrix kernels require positive dimensions and strides") | |
| if algorithm == "aten": | |
| return torch.mm(a, b) | |
| bm, bn, bk, warps, stages = normalize_tile(tile) | |
| m, k = a.shape | |
| n = b.shape[1] | |
| if algorithm == "persistent_tma": | |
| if (min(a.stride()) != 1 or min(b.stride()) != 1 | |
| or max(a.stride()) * 2 % 16 or max(b.stride()) * 2 % 16): | |
| raise ValueError("TMA operands require an innermost stride of one and 16-byte-aligned rows") | |
| if a.storage_offset() * 2 % 16 or b.storage_offset() * 2 % 16: | |
| raise ValueError("TMA operands require 16-byte-aligned storage offsets") | |
| return _tma_mm(a, b, bm, bn, bk, warps, stages) | |
| output = torch.empty((m, n), device=a.device, dtype=torch.bfloat16) | |
| tiles = triton.cdiv(m, bm) * triton.cdiv(n, bn) | |
| programs = min(tiles, torch.cuda.get_device_properties(a.device).multi_processor_count) \ | |
| if algorithm == "persistent" else tiles | |
| wrap_triton(_matmul_kernel)[(programs,)]( | |
| a, b, output, m, n, k, a.stride(0), a.stride(1), b.stride(0), b.stride(1), | |
| bm, bn, bk, 8, programs, algorithm == "persistent", | |
| num_warps=warps, num_stages=stages) | |
| return output | |