WarpStateAI / test_matmul_kernels.py
zerdovzad's picture
Upload 139 files
c8ba9c6 verified
Raw History Blame Contribute Delete
8.12 kB
"""Matrix tile validation, dispatch signatures, and explicit CUDA correctness."""
import json
import os
import pytest
import torch
from torch.library import triton_op
from ops.matmul_kernels import (
DEFAULT_TILE, TUNING_TILES, launch_mm, matrix_signature, normalize_tile,
policy_key, tile_dict,
)
@pytest.mark.parametrize("tile", [
[], (128, 128, 32, 4), (128, 128, 32, True, 3), (120, 128, 32, 4, 3),
(128, 128, 32, 2, 3), (128, 128, 32, 4, 8), (256, 256, 128, 8, 4),
{"block_m": 128},
])
def test_invalid_or_oversized_tiles_fail(tile):
with pytest.raises(ValueError):
normalize_tile(tile)
def test_tuning_tiles_and_json_round_trip():
for tile in TUNING_TILES:
assert normalize_tile(json.loads(json.dumps(tile_dict(tile)))) == tile
assert normalize_tile() == DEFAULT_TILE
def test_signature_distinguishes_strides_even_when_shapes_match():
a = torch.empty((16, 32), dtype=torch.bfloat16)
b = torch.empty((64, 32), dtype=torch.bfloat16).T
signature = matrix_signature(a, b)
assert signature == {"m": 16, "n": 64, "k": 32, "a_stride": [32, 1],
"b_stride": [1, 32], "dtype": "bfloat16"}
assert json.loads(policy_key(a, b)) == signature
assert policy_key(a, b) != policy_key(a, b.contiguous())
def test_no_cpu_or_unknown_algorithm_fallback():
a = torch.empty((16, 32), dtype=torch.bfloat16)
b = torch.empty((32, 16), dtype=torch.bfloat16)
with pytest.raises(ValueError, match="CUDA"):
launch_mm(a, b)
with pytest.raises(ValueError, match="Unknown"):
launch_mm(a, b, "fp8")
def test_tuner_kind_defaults_and_selected_tiles():
from ops.tune_matmul import arguments, cases
generic = arguments(["--list-cases"])
assert generic.operations == ["fwd", "dw"]
assert generic.algorithms == ["tiled", "persistent", "persistent_tma"]
fused = arguments(["--list-cases", "--kind", "swiglu", "--tile", "128,128,32,8,3"])
assert fused.operations == ["fwd"]
assert fused.algorithms == ["tiled", "persistent"]
assert fused.tile == [(128, 128, 32, 8, 3)]
forward, dw = cases(8192, generic.operations)
assert forward["a_stride"] == [2048, 1] and forward["b_stride"] == [1, 2048]
assert dw["a_stride"] == [1, 14336] and dw["b_stride"] == [2048, 1]
@pytest.mark.parametrize("flags", [
["--kind", "swiglu", "--operations", "dw"],
["--kind", "swiglu", "--algorithms", "persistent_tma"],
["--kind", "swiglu", "--tile", "256,256,32,8,3"],
["--tile", "128,128,32,4"],
["--input-scale", "nan"], ["--input-scale", "0"],
["--operations", "fwd", "fwd"],
["--algorithms", "tiled", "tiled"],
["--tile", "128,128,32,4,3", "--tile", "128,128,32,4,3"],
["--output-dir", "/tmp/outside-workspace"],
])
def test_tuner_rejects_unsupported_kind_or_unsafe_output(flags):
from ops.tune_matmul import arguments
with pytest.raises(SystemExit):
arguments(["--list-cases", *flags])
def test_tuner_checks_all_outputs_and_preserves_bf16():
from ops.tune_matmul import output_metrics
expected = torch.ones((4, 8), dtype=torch.bfloat16)
metrics = output_metrics((expected.clone(), expected * 2), (expected, expected))
assert metrics["output_metrics"][0]["relative_rms_error"] == 0
assert metrics["relative_rms_error"] == 1
with pytest.raises(ValueError, match="BF16"):
output_metrics(expected.float(), expected)
with pytest.raises(ValueError, match="Nonfinite"):
output_metrics((expected, expected * float("nan")), (expected, expected))
def test_tuner_cannot_select_a_median_winner_with_a_losing_trial():
from ops.tune_matmul import paired_timing_metrics
trials = [{"aten_before_ms": 2.0, "candidate_ms": candidate, "aten_after_ms": 2.0}
for candidate in (1.5, 1.5, 2.1)]
metrics = paired_timing_metrics(trials, minimum_gain=0.01)
assert metrics["paired_gain"] > 0.3
assert metrics["minimum_trial_gain"] < 0
assert not metrics["timing_eligible"]
drifted = [{"aten_before_ms": 1.0, "candidate_ms": 1.4, "aten_after_ms": 2.0}]
metrics = paired_timing_metrics(drifted, minimum_gain=0.01)
assert metrics["timing_eligible"]
assert metrics["maximum_control_drift_fraction"] == pytest.approx(2 / 3)
@pytest.mark.parametrize("timing", [0, -1, float("nan"), float("inf")])
def test_tuner_rejects_invalid_timing_metrics(timing):
from ops.tune_matmul import paired_timing_metrics
with pytest.raises(ValueError, match="finite and positive"):
paired_timing_metrics([{"aten_before_ms": 1, "candidate_ms": timing,
"aten_after_ms": 1}], minimum_gain=0.01)
@pytest.mark.parametrize("offset_operand", ["a", "b"])
def test_tma_rejects_misaligned_offset_without_launching(offset_operand):
from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode
mode = FakeTensorMode()
# Construct fake CUDA operands from meta storage; this test never starts
# CUDA or reaches the custom-op launch, including on CPU-only machines.
tensors = []
for name, shape in (("a", (32, 64)), ("b", (64, 32))):
offset = 1 if name == offset_operand else 0
storage = torch.empty(shape[0] * shape[1] + offset, device="meta", dtype=torch.bfloat16)
view = storage.as_strided(shape, (shape[1], 1), offset)
tensors.append(FakeTensor(mode, view, torch.device("cuda:0")))
with pytest.raises(ValueError, match="16-byte-aligned storage offsets"):
launch_mm(*tensors, "persistent_tma")
cuda_window = pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1",
reason="requires an explicitly reserved CUDA device")
@triton_op("warpstate_test::candidate_mm", mutates_args={})
def _test_mm(a: torch.Tensor, b: torch.Tensor, algorithm: str) -> torch.Tensor:
return launch_mm(a, b, algorithm, DEFAULT_TILE)
@cuda_window
@pytest.mark.parametrize("algorithm", ["tiled", "persistent"])
@pytest.mark.parametrize("layouts", [(False, False), (False, True), (True, False), (True, True)])
def test_cuda_odd_shapes_and_transposed_operands(algorithm, layouts):
torch.manual_seed(77)
m, n, k = 97, 113, 129
a = torch.randn((k, m) if layouts[0] else (m, k), device="cuda", dtype=torch.bfloat16)
b = torch.randn((n, k) if layouts[1] else (k, n), device="cuda", dtype=torch.bfloat16)
a = a.T if layouts[0] else a
b = b.T if layouts[1] else b
acopy, bcopy = a.clone(), b.clone()
with torch.inference_mode():
expected = torch.mm(a, b)
output = launch_mm(a, b, algorithm)
error = (output.float() - expected.float()).square().mean().sqrt()
assert output.dtype == torch.bfloat16 and torch.isfinite(output).all()
assert error / expected.float().square().mean().sqrt() < 0.003
torch.testing.assert_close(a, acopy, rtol=0, atol=0)
torch.testing.assert_close(b, bcopy, rtol=0, atol=0)
@cuda_window
@pytest.mark.parametrize("algorithm", ["tiled", "persistent", "persistent_tma"])
def test_cuda_launch_is_usable_under_compiled_triton_op(algorithm):
a = torch.randn((128, 64), device="cuda", dtype=torch.bfloat16)
b = torch.randn((96, 64), device="cuda", dtype=torch.bfloat16).T
compiled = torch.compile(lambda x, y: _test_mm(x, y, algorithm), fullgraph=True)
torch.testing.assert_close(compiled(a, b), _test_mm(a, b, algorithm), rtol=0, atol=0)
@cuda_window
@pytest.mark.parametrize("layouts", [(False, False), (False, True), (True, False), (True, True)])
def test_cuda_tma_layouts_with_masked_output_tiles(layouts):
m, n, k = 96, 112, 128
a = torch.randn((k, m) if layouts[0] else (m, k), device="cuda", dtype=torch.bfloat16)
b = torch.randn((n, k) if layouts[1] else (k, n), device="cuda", dtype=torch.bfloat16)
a, b = a.T if layouts[0] else a, b.T if layouts[1] else b
expected = torch.mm(a, b)
actual = launch_mm(a, b, "persistent_tma")
difference = actual.float() - expected.float()
assert torch.isfinite(actual).all()
assert difference.square().mean().sqrt() / expected.float().square().mean().sqrt() < 0.003