File size: 8,122 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
"""Matrix tile validation, dispatch signatures, and explicit CUDA correctness."""

import json
import os

import pytest
import torch
from torch.library import triton_op

from ops.matmul_kernels import (
    DEFAULT_TILE, TUNING_TILES, launch_mm, matrix_signature, normalize_tile,
    policy_key, tile_dict,
)


@pytest.mark.parametrize("tile", [
    [], (128, 128, 32, 4), (128, 128, 32, True, 3), (120, 128, 32, 4, 3),
    (128, 128, 32, 2, 3), (128, 128, 32, 4, 8), (256, 256, 128, 8, 4),
    {"block_m": 128},
])
def test_invalid_or_oversized_tiles_fail(tile):
    with pytest.raises(ValueError):
        normalize_tile(tile)


def test_tuning_tiles_and_json_round_trip():
    for tile in TUNING_TILES:
        assert normalize_tile(json.loads(json.dumps(tile_dict(tile)))) == tile
    assert normalize_tile() == DEFAULT_TILE


def test_signature_distinguishes_strides_even_when_shapes_match():
    a = torch.empty((16, 32), dtype=torch.bfloat16)
    b = torch.empty((64, 32), dtype=torch.bfloat16).T
    signature = matrix_signature(a, b)
    assert signature == {"m": 16, "n": 64, "k": 32, "a_stride": [32, 1],
                         "b_stride": [1, 32], "dtype": "bfloat16"}
    assert json.loads(policy_key(a, b)) == signature
    assert policy_key(a, b) != policy_key(a, b.contiguous())


def test_no_cpu_or_unknown_algorithm_fallback():
    a = torch.empty((16, 32), dtype=torch.bfloat16)
    b = torch.empty((32, 16), dtype=torch.bfloat16)
    with pytest.raises(ValueError, match="CUDA"):
        launch_mm(a, b)
    with pytest.raises(ValueError, match="Unknown"):
        launch_mm(a, b, "fp8")


def test_tuner_kind_defaults_and_selected_tiles():
    from ops.tune_matmul import arguments, cases

    generic = arguments(["--list-cases"])
    assert generic.operations == ["fwd", "dw"]
    assert generic.algorithms == ["tiled", "persistent", "persistent_tma"]
    fused = arguments(["--list-cases", "--kind", "swiglu", "--tile", "128,128,32,8,3"])
    assert fused.operations == ["fwd"]
    assert fused.algorithms == ["tiled", "persistent"]
    assert fused.tile == [(128, 128, 32, 8, 3)]
    forward, dw = cases(8192, generic.operations)
    assert forward["a_stride"] == [2048, 1] and forward["b_stride"] == [1, 2048]
    assert dw["a_stride"] == [1, 14336] and dw["b_stride"] == [2048, 1]


@pytest.mark.parametrize("flags", [
    ["--kind", "swiglu", "--operations", "dw"],
    ["--kind", "swiglu", "--algorithms", "persistent_tma"],
    ["--kind", "swiglu", "--tile", "256,256,32,8,3"],
    ["--tile", "128,128,32,4"],
    ["--input-scale", "nan"], ["--input-scale", "0"],
    ["--operations", "fwd", "fwd"],
    ["--algorithms", "tiled", "tiled"],
    ["--tile", "128,128,32,4,3", "--tile", "128,128,32,4,3"],
    ["--output-dir", "/tmp/outside-workspace"],
])
def test_tuner_rejects_unsupported_kind_or_unsafe_output(flags):
    from ops.tune_matmul import arguments

    with pytest.raises(SystemExit):
        arguments(["--list-cases", *flags])


def test_tuner_checks_all_outputs_and_preserves_bf16():
    from ops.tune_matmul import output_metrics

    expected = torch.ones((4, 8), dtype=torch.bfloat16)
    metrics = output_metrics((expected.clone(), expected * 2), (expected, expected))
    assert metrics["output_metrics"][0]["relative_rms_error"] == 0
    assert metrics["relative_rms_error"] == 1
    with pytest.raises(ValueError, match="BF16"):
        output_metrics(expected.float(), expected)
    with pytest.raises(ValueError, match="Nonfinite"):
        output_metrics((expected, expected * float("nan")), (expected, expected))


def test_tuner_cannot_select_a_median_winner_with_a_losing_trial():
    from ops.tune_matmul import paired_timing_metrics

    trials = [{"aten_before_ms": 2.0, "candidate_ms": candidate, "aten_after_ms": 2.0}
              for candidate in (1.5, 1.5, 2.1)]
    metrics = paired_timing_metrics(trials, minimum_gain=0.01)
    assert metrics["paired_gain"] > 0.3
    assert metrics["minimum_trial_gain"] < 0
    assert not metrics["timing_eligible"]
    drifted = [{"aten_before_ms": 1.0, "candidate_ms": 1.4, "aten_after_ms": 2.0}]
    metrics = paired_timing_metrics(drifted, minimum_gain=0.01)
    assert metrics["timing_eligible"]
    assert metrics["maximum_control_drift_fraction"] == pytest.approx(2 / 3)


@pytest.mark.parametrize("timing", [0, -1, float("nan"), float("inf")])
def test_tuner_rejects_invalid_timing_metrics(timing):
    from ops.tune_matmul import paired_timing_metrics

    with pytest.raises(ValueError, match="finite and positive"):
        paired_timing_metrics([{"aten_before_ms": 1, "candidate_ms": timing,
                                "aten_after_ms": 1}], minimum_gain=0.01)


@pytest.mark.parametrize("offset_operand", ["a", "b"])
def test_tma_rejects_misaligned_offset_without_launching(offset_operand):
    from torch._subclasses.fake_tensor import FakeTensor, FakeTensorMode

    mode = FakeTensorMode()
    # Construct fake CUDA operands from meta storage; this test never starts
    # CUDA or reaches the custom-op launch, including on CPU-only machines.
    tensors = []
    for name, shape in (("a", (32, 64)), ("b", (64, 32))):
        offset = 1 if name == offset_operand else 0
        storage = torch.empty(shape[0] * shape[1] + offset, device="meta", dtype=torch.bfloat16)
        view = storage.as_strided(shape, (shape[1], 1), offset)
        tensors.append(FakeTensor(mode, view, torch.device("cuda:0")))
    with pytest.raises(ValueError, match="16-byte-aligned storage offsets"):
        launch_mm(*tensors, "persistent_tma")


cuda_window = pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1",
                                 reason="requires an explicitly reserved CUDA device")


@triton_op("warpstate_test::candidate_mm", mutates_args={})
def _test_mm(a: torch.Tensor, b: torch.Tensor, algorithm: str) -> torch.Tensor:
    return launch_mm(a, b, algorithm, DEFAULT_TILE)


@cuda_window
@pytest.mark.parametrize("algorithm", ["tiled", "persistent"])
@pytest.mark.parametrize("layouts", [(False, False), (False, True), (True, False), (True, True)])
def test_cuda_odd_shapes_and_transposed_operands(algorithm, layouts):
    torch.manual_seed(77)
    m, n, k = 97, 113, 129
    a = torch.randn((k, m) if layouts[0] else (m, k), device="cuda", dtype=torch.bfloat16)
    b = torch.randn((n, k) if layouts[1] else (k, n), device="cuda", dtype=torch.bfloat16)
    a = a.T if layouts[0] else a
    b = b.T if layouts[1] else b
    acopy, bcopy = a.clone(), b.clone()
    with torch.inference_mode():
        expected = torch.mm(a, b)
        output = launch_mm(a, b, algorithm)
    error = (output.float() - expected.float()).square().mean().sqrt()
    assert output.dtype == torch.bfloat16 and torch.isfinite(output).all()
    assert error / expected.float().square().mean().sqrt() < 0.003
    torch.testing.assert_close(a, acopy, rtol=0, atol=0)
    torch.testing.assert_close(b, bcopy, rtol=0, atol=0)


@cuda_window
@pytest.mark.parametrize("algorithm", ["tiled", "persistent", "persistent_tma"])
def test_cuda_launch_is_usable_under_compiled_triton_op(algorithm):
    a = torch.randn((128, 64), device="cuda", dtype=torch.bfloat16)
    b = torch.randn((96, 64), device="cuda", dtype=torch.bfloat16).T
    compiled = torch.compile(lambda x, y: _test_mm(x, y, algorithm), fullgraph=True)
    torch.testing.assert_close(compiled(a, b), _test_mm(a, b, algorithm), rtol=0, atol=0)


@cuda_window
@pytest.mark.parametrize("layouts", [(False, False), (False, True), (True, False), (True, True)])
def test_cuda_tma_layouts_with_masked_output_tiles(layouts):
    m, n, k = 96, 112, 128
    a = torch.randn((k, m) if layouts[0] else (m, k), device="cuda", dtype=torch.bfloat16)
    b = torch.randn((n, k) if layouts[1] else (k, n), device="cuda", dtype=torch.bfloat16)
    a, b = a.T if layouts[0] else a, b.T if layouts[1] else b
    expected = torch.mm(a, b)
    actual = launch_mm(a, b, "persistent_tma")
    difference = actual.float() - expected.float()
    assert torch.isfinite(actual).all()
    assert difference.square().mean().sqrt() / expected.float().square().mean().sqrt() < 0.003