File size: 6,848 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
"""Independent derivative/index checks for the isolated backward layout prototype."""

import os

import pytest
import torch

from ops.backward_layout import (PROFILE_SHAPE, launch_layout, make_inputs,
                                 prepare_layout, reference_layout, validate_inputs)


def token_major(value):
    b, n, h, c, d = value.shape
    return value.permute(0, 1, 3, 2, 4).reshape(b, n * c, h * d)


def test_pair_direction_and_projection_packing():
    inputs = make_inputs((1, 2, 2, 3, 4))
    value = torch.arange(1, 49, dtype=torch.bfloat16).reshape(inputs.q_memory.shape)
    inputs = inputs._replace(q_memory=value, q_local=torch.zeros_like(value),
        k_write=torch.zeros_like(inputs.k_write), k_local=value, v_local=value + 1,
        sin=torch.ones_like(inputs.sin), cos=torch.zeros_like(inputs.cos))
    q, k, v, _, _ = reference_layout(inputs).chunk(5, -1)
    wanted = token_major(torch.stack((value[..., 1::2], -value[..., ::2]), -1).flatten(-2))
    assert torch.equal(q, wanted) and torch.equal(k, wanted)
    assert torch.equal(v, token_major(value + 1))


def test_q_matches_model_rope_vjp_with_unequal_pair_coefficients():
    from warpstate.model import apply_rope
    inputs = make_inputs((2, 3, 2, 7, 6), seed=921)
    generator = torch.Generator().manual_seed(799)
    inputs = inputs._replace(sin=torch.randn(inputs.sin.shape, generator=generator),
                             cos=torch.randn(inputs.cos.shape, generator=generator))
    original = torch.zeros_like(inputs.q_memory, dtype=torch.float32, requires_grad=True)
    output = apply_rope(original, inputs.cos, inputs.sin)
    gradient, = torch.autograd.grad(output, original,
        grad_outputs=inputs.q_memory.float() + inputs.q_local.float())
    assert torch.equal(reference_layout(inputs).chunk(5, -1)[0], token_major(gradient.bfloat16()))


def test_k_intermediate_bf16_rounding_is_observable():
    inputs = make_inputs((1, 2, 2, 7, 16), seed=19)
    k = (inputs.k_write.transpose(-2, -1).float()
         * (1 - inputs.tanh_k.float().square()) + inputs.k_local.float())
    sine = k * inputs.sin
    unrounded_rotary = torch.stack((sine[..., 1::2], -sine[..., ::2]), -1).flatten(-2)
    wrong = token_major((unrounded_rotary + k * inputs.cos).bfloat16())
    assert torch.any(reference_layout(inputs).chunk(5, -1)[1] != wrong)


def test_gate_matches_independent_autograd_and_u_packing():
    inputs = make_inputs((1, 2, 2, 3, 4), seed=987)
    inputs = inputs._replace(tanh_u=torch.zeros_like(inputs.tanh_u))
    gate = inputs.gate.float().requires_grad_()
    blended = gate * inputs.local.float() + (1 - gate) * inputs.memory.float() / 2
    dy = inputs.grad_y.reshape(1, 2, 3, 2, 4).permute(0, 1, 3, 2, 4).float()
    derivative, = torch.autograd.grad(blended, gate, grad_outputs=dy)
    expected = token_major((derivative * gate * (1 - gate)).bfloat16())
    _, _, _, actual, u = reference_layout(inputs).chunk(5, -1)
    torch.testing.assert_close(actual, expected, rtol=0.01, atol=1e-6)
    assert torch.equal(u, token_major(inputs.u_write))


@pytest.mark.parametrize("case", ["dtype", "k_shape", "rope_dtype", "strides", "device", "odd_head"])
def test_contract_rejects_unsupported_inputs(case):
    inputs = make_inputs((1, 2, 2, 3, 4))
    if case == "dtype":
        inputs = inputs._replace(q_memory=inputs.q_memory.float())
    elif case == "k_shape":
        inputs = inputs._replace(k_write=inputs.k_write[..., :-1])
    elif case == "rope_dtype":
        inputs = inputs._replace(sin=inputs.sin.bfloat16())
    elif case == "strides":
        inputs = inputs._replace(v_local=inputs.v_local.transpose(1, 2))
    elif case == "device":
        inputs = inputs._replace(gate=torch.empty(inputs.gate.shape, device="meta", dtype=torch.bfloat16))
    else:
        inputs = inputs._replace(q_memory=inputs.q_memory[..., :-1])
    with pytest.raises(ValueError):
        validate_inputs(inputs)


def test_cuda_launch_contract_rejects_cpu():
    with pytest.raises(ValueError, match="require CUDA"):
        prepare_layout(make_inputs((1, 1, 1, 1, 2)))


def test_benchmark_distinguishes_signed_zero_and_serializes_nonfinite_outputs():
    import json
    from ops.backward_layout_benchmark import output_errors
    expected = torch.zeros((1, 1, 10), dtype=torch.bfloat16)
    actual = expected.clone()
    actual[0, 0, 0] = -0.0
    report = output_errors(actual, expected)
    assert report["q"]["unequal"] == 0
    assert report["q"]["bitwise_unequal"] == 1
    actual[0, 0, 3] = float("nan")
    report = output_errors(actual, expected)
    assert not report["k"]["finite"] and report["k"]["relative_rms"] is None
    json.dumps(report, allow_nan=False)


cuda_window = pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1",
                                reason="requires an explicitly reserved CUDA device")


@cuda_window
@pytest.mark.parametrize("algorithm", ["paired", "single"])
@pytest.mark.parametrize("shape", [(1, 2, 3, 7, 10), (2, 3, 2, 16, 64)])
def test_cuda_ragged_and_production_head_arithmetic(algorithm, shape):
    inputs = make_inputs(shape, device="cuda", seed=455)
    # Distinct pair coefficients detect accidentally selecting the wrong RoPE lane.
    inputs = inputs._replace(sin=torch.randn_like(inputs.sin), cos=torch.randn_like(inputs.cos))
    saved = [value.clone() for value in inputs]
    actual = launch_layout(inputs, algorithm, block=256)
    expected = reference_layout(inputs)
    assert actual.dtype == torch.bfloat16 and actual.is_contiguous()
    for value, wanted in zip(actual.chunk(5, -1), expected.chunk(5, -1), strict=True):
        assert torch.isfinite(value).all()
        error = (value.float() - wanted.float()).norm() / wanted.float().norm().clamp_min(1e-20)
        assert float(error) < 0.002
        torch.testing.assert_close(value, wanted, rtol=0.015, atol=1e-4)
    assert all(torch.equal(value, original) for value, original in zip(inputs, saved, strict=True))


@cuda_window
@pytest.mark.parametrize("algorithm", ["paired", "single"])
def test_cuda_production_shape_matches_exact_profile_kernels(algorithm):
    inputs = make_inputs(PROFILE_SHAPE, device="cuda", seed=817)
    expected = launch_layout(inputs, "profile")
    actual = launch_layout(inputs, algorithm, block=512)
    for value, wanted in zip(actual.chunk(5, -1), expected.chunk(5, -1), strict=True):
        assert torch.isfinite(value).all()
        assert torch.equal(value.view(torch.int16), wanted.view(torch.int16))


@cuda_window
def test_cuda_profile_scratch_cannot_alias():
    inputs = make_inputs(PROFILE_SHAPE, device="cuda")
    scratch = torch.empty_like(inputs.q_memory)
    for values in ((scratch, scratch), (inputs.q_local, scratch)):
        with pytest.raises(ValueError, match="Scratch tensors may not alias"):
            prepare_layout(inputs, "profile", scratch=values)