File size: 12,228 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
"""Structural and configuration guards for the opt-in backward graph rewrite.

The pattern fixtures isolate graph safety; actual compiled model numerical and
continuation checks live in the separate integration validation harness.
"""

import copy
import hashlib
import operator
import os
from pathlib import Path

import pytest
import torch
from torch._inductor import config

from ops import backward_layout_execution as layout
from warpstate.config import WarpStateConfig


@pytest.fixture(autouse=True)
def restore_configuration(monkeypatch):
    monkeypatch.setattr(layout, "_ACTIVE_PASS", None)
    monkeypatch.setattr(layout, "_ACTIVE_MODE", "baseline")
    monkeypatch.setattr(layout, "_COUNTS", layout.Counter())
    monkeypatch.setattr(layout, "_MISSES", layout.Counter())
    with config.patch({"post_grad_custom_post_pass": None, "emulate_precision_casts": False,
                       "triton.codegen_upcast_to_fp32": True, "eager_numerics.disable_ftz": False}):
        yield


def candidate_graph():
    """Attach explicit native GEMM and ordered Flash VJP boundary producers."""
    gm = copy.deepcopy(layout._pattern())
    graph = gm.graph
    placeholders = [node for node in graph.nodes if node.op == "placeholder"]
    values = dict(zip(layout._NAMES, placeholders, strict=True))
    first = next(node for node in graph.nodes if node.op == "call_function")
    with graph.inserting_before(first):
        flash = graph.call_function(torch.ops.aten._scaled_dot_product_flash_attention_backward.default,
                                    (None, None, None, None, values["local"], None, None, None,
                                     128, 128, 0.0, True, None, None), {"scale": 0.125})
        for name, index in (("q_local", 0), ("k_local", 1), ("v_local", 2)):
            old = values[name]
            new = graph.call_function(operator.getitem, (flash, index))
            new.meta = dict(old.meta)
            old.replace_all_uses_with(new)
        for name in ("q_memory", "k_write", "u_write", "grad_y"):
            old = values[name]
            users = list(old.users)
            target = torch.ops.aten.mm.default if name == "grad_y" else torch.ops.aten.bmm.default
            new = graph.call_function(target, (old, old))
            new.meta = dict(old.meta)
            for user in users:
                user.replace_input_with(old, new)
    graph.lint()
    root = next(node for node in graph.nodes if node.op == "output").args[0]
    return gm, root, flash


def test_pattern_is_independent_of_node_names_and_preserves_native_inputs():
    gm, root, _ = candidate_graph()
    for index, node in enumerate(gm.graph.nodes):
        node.name = f"renamed_{index}"
    inputs, interior = layout._match(root, layout._pattern())
    assert len(inputs) == 14
    assert len(interior) > 70
    assert tuple(inputs[4].meta["val"].shape) == (2048, 64, 128)
    assert tuple(inputs[8].meta["val"].shape) == (8192, 2048)


def test_inspection_retains_graph_and_replacement_is_pure():
    gm, root, _ = candidate_graph()
    original = str(gm.graph)
    layout.BackwardLayoutPass("inspect", "inspect-test")(gm.graph)
    assert str(gm.graph) == original
    assert layout.backward_layout_stats()["matches"] == 1
    layout.BackwardLayoutPass("paired", "paired-test")(gm.graph)
    gm.graph.lint()
    calls = [node for node in gm.graph.nodes
             if node.target == torch.ops.warpstate.paired_backward_layout.default]
    assert len(calls) == 1
    assert root not in gm.graph.nodes
    assert len(calls[0].args) == 14
    assert layout.backward_layout_stats()["replacements"] == 1
    assert layout.backward_layout_stats()["runtime_calls"] == 0


@pytest.mark.parametrize("mutation", ["gate_scale", "projection_order", "rotary_lane", "rounding",
                                      "dtype", "stride", "dropout", "flash_scale", "flash_order",
                                      "input_producer", "missing_meta", "mixed_devices"])
def test_unsupported_graph_is_left_unchanged(mutation):
    gm, root, flash = candidate_graph()
    nodes = list(gm.graph.nodes)
    if mutation == "gate_scale":
        node = next(n for n in nodes if n.target == torch.ops.aten.div.Tensor)
        node.args = (node.args[0], 16.0)
    elif mutation == "projection_order":
        projections = list(root.args[0]); projections[0], projections[1] = projections[1], projections[0]
        root.args = (projections, 2)
    elif mutation == "rotary_lane":
        node = next(n for n in nodes if n.target == torch.ops.aten.select.int)
        node.args = (*node.args[:-1], 1 - node.args[-1])
    elif mutation == "rounding":
        node = next(n for n in nodes if n.target == torch.ops.prims.convert_element_type.default)
        node.args = (node.args[0], torch.float32)
    elif mutation in ("dtype", "stride", "missing_meta"):
        if mutation == "missing_meta":
            root.meta.pop("val")
        elif mutation == "dtype":
            root.meta["val"] = root.meta["val"].float()
        else:
            root.meta["val"] = root.meta["val"].transpose(1, 2).contiguous().transpose(1, 2)
            assert tuple(root.meta["val"].shape) == (4, 2048, 10240)
    elif mutation == "mixed_devices":
        from torch._subclasses.fake_tensor import FakeTensor

        node = next(n for n in nodes if n.target == torch.ops.aten.bmm.default)
        value = node.meta["val"]
        node.meta["val"] = FakeTensor(value.fake_mode,
                                      torch.empty_strided(value.shape, value.stride(),
                                                          dtype=value.dtype, device="meta"),
                                      torch.device("cuda:1"))
    elif mutation == "dropout":
        args = list(flash.args); args[10] = 0.1; flash.args = tuple(args)
    elif mutation == "flash_scale":
        flash.kwargs = {"scale": 0.25}
    elif mutation == "flash_order":
        node = next(n for n in nodes if n.target is operator.getitem)
        node.args = (flash, 2)
    else:
        node = next(n for n in nodes if n.target == torch.ops.aten.bmm.default)
        node.target = torch.ops.aten.mm.default
    original = str(gm.graph)
    layout.BackwardLayoutPass("paired", "test")(gm.graph)
    assert str(gm.graph) == original
    assert layout.backward_layout_stats()["replacements"] == 0
    assert sum(layout.backward_layout_stats()["miss_reasons"].values()) == 1


def test_external_arithmetic_consumer_rejected():
    gm, root, _ = candidate_graph()
    node = next(n for n in gm.graph.nodes if n.target == torch.ops.aten.add.Tensor)
    with gm.graph.inserting_before(root):
        gm.graph.call_function(torch.ops.aten.neg.default, (node,))
    with pytest.raises(layout._Mismatch, match="external intermediate consumer"):
        layout._match(root, layout._pattern())


def test_shared_gradient_view_retained_without_copy():
    gm, root, _ = candidate_graph()
    node = next(n for n in gm.graph.nodes if n.target == torch.ops.aten.permute.default
                and tuple(n.meta["val"].stride()) == (4194304, 262144, 64, 2048, 1))
    with gm.graph.inserting_before(root):
        external = gm.graph.call_function(torch.ops.aten.neg.default, (node,))
    _, interior = layout._match(root, layout._pattern())
    assert node not in interior
    layout.BackwardLayoutPass("paired", "test")(gm.graph)
    assert external.args[0] is node
    gm.graph.lint()


def test_cse_shared_exact_zero_across_two_regions():
    from torch._functorch.compile_utils import fx_graph_cse

    graph = torch.fx.Graph()
    regions = [candidate_graph(), candidate_graph()]
    mappings = []
    for index, (gm, _, _) in enumerate(regions):
        mapping = {}
        for node in gm.graph.nodes:
            if node.op == "placeholder":
                new = graph.placeholder(f"core_{index}_{node.name}")
                new.meta = dict(node.meta)
                mapping[node] = new
        mappings.append(mapping)
    roots = []
    for (gm, root, _), mapping in zip(regions, mappings, strict=True):
        for node in gm.graph.nodes:
            if node.op not in ("placeholder", "output"):
                mapping[node] = graph.node_copy(node, lambda value: mapping[value])
        roots.append(mapping[root])
    graph.output(tuple(roots))
    graph = fx_graph_cse(graph)
    zeros = [node for node in graph.nodes if node.target == torch.ops.aten.full.default]
    assert len(zeros) == 1 and len(zeros[0].users) == 8
    layout.BackwardLayoutPass("paired", "test")(graph)
    graph.lint()
    assert sum(node.target == torch.ops.warpstate.paired_backward_layout.default
               for node in graph.nodes) == 2
    assert not any(node.target == torch.ops.aten.full.default for node in graph.nodes)
    assert layout.backward_layout_stats()["replacements"] == 2


def test_configuration_identity_covers_sources():
    result = layout.configure_backward_layout("paired")
    assert config.post_grad_custom_post_pass.uuid() == result["identity"]
    for name, digest in result["source_sha256"].items():
        assert hashlib.sha256((Path(__file__).parent / name).read_bytes()).hexdigest() == digest
    with pytest.raises(RuntimeError, match="already configured"):
        layout.configure_backward_layout("paired")


def test_configuration_rejects_existing_pass_and_precision_changes():
    existing = lambda graph: None
    with config.patch(post_grad_custom_post_pass=existing):
        with pytest.raises(RuntimeError, match="already exists"):
            layout.configure_backward_layout("paired")
        assert config.post_grad_custom_post_pass is existing
    for setting, value in (("emulate_precision_casts", True), ("triton.codegen_upcast_to_fp32", False),
                           ("eager_numerics.disable_ftz", True)):
        with config.patch({setting: value}):
            with pytest.raises(ValueError, match="FP32 pointwise"):
                layout.configure_backward_layout("paired")


def test_baseline_has_no_compiler_hook_or_metadata():
    assert layout.configure_backward_layout() is None
    assert config.post_grad_custom_post_pass is None


def test_dispatch_evidence_requires_actual_runtime_calls():
    layout.configure_backward_layout("paired")
    layout._COUNTS["replacements"] = 100
    with pytest.raises(RuntimeError, match="runtime calls"):
        layout.require_backward_layout_dispatch()
    layout._COUNTS["runtime_calls"] = 2
    assert layout.require_backward_layout_dispatch(2)["runtime_calls"] == 2
    with pytest.raises(RuntimeError, match="at least 3"):
        layout.require_backward_layout_dispatch(3)
    with pytest.raises(ValueError):
        layout.require_backward_layout_dispatch(True)


def test_native_entry_rejects_unsupported_inputs_before_launch():
    with pytest.raises(ValueError, match="q_memory"):
        layout._validate_native_inputs([torch.empty(1)] * 14)


@pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1" or not torch.cuda.is_available(),
                    reason="requires an explicitly reserved CUDA device")
def test_opaque_boundary_schema_and_fake_registration():
    from ops.backward_layout import make_inputs

    saved = make_inputs(device="cuda", seed=9221)
    native = tuple(value.view(shape) for value, shape in
                   zip(saved, layout._NATIVE_SHAPES, strict=True))
    # This is a backward-only internal operator; schema purity and accurate fake
    # metadata apply, whereas registering another autograd VJP would be wrong.
    result = torch.library.opcheck(torch.ops.warpstate.paired_backward_layout.default, native,
                                   test_utils=("test_schema", "test_faketensor"))
    assert set(result.values()) == {"SUCCESS"}


@pytest.mark.parametrize("field,value", [("d_model", 1024), ("n_heads", 16), ("chunk_size", 64),
                                        ("parallel_memory", False), ("dropout", 0.1)])
def test_model_configuration_guard(field, value):
    cfg = WarpStateConfig(d_model=2048, n_heads=32, chunk_size=128, parallel_memory=True)
    layout.validate_backward_layout_config(cfg)
    setattr(cfg, field, value)
    with pytest.raises(ValueError, match=field):
        layout.validate_backward_layout_config(cfg)