File size: 7,502 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
"""Optional activation kernels preserve state, rounding and usable gradients."""

from __future__ import annotations

import copy
import os

import pytest
import torch
import torch.nn.functional as F

from ops.activation_execution import apply_activation_execution, bf16_rmsnorm, bf16_swiglu
from warpstate import WarpStateConfig, WarpStateLM
from warpstate.model import RMSNorm


def _model():
    torch.manual_seed(943)
    return WarpStateLM(WarpStateConfig(vocab_size=48, d_model=32, n_heads=2,
        n_cores=1, logical_depth=2, ffn_hidden=56, chunk_size=4,
        gradient_checkpointing=False, parallel_memory=True)).train()


@pytest.mark.parametrize("mode", ["baseline", "swiglu", "rmsnorm", "all"])
@pytest.mark.parametrize("autocast", [False, True])
def test_cpu_fallback_retains_parameters_state_logits_and_gradients(mode, autocast):
    reference = _model()
    actual = copy.deepcopy(reference)
    parameters = tuple(actual.parameters())
    keys = tuple(actual.state_dict())
    apply_activation_execution(actual, mode)
    tokens = torch.randint(48, (2, 8))
    with torch.autocast("cpu", dtype=torch.bfloat16, enabled=autocast):
        expected = reference(tokens)
        result = actual(tokens)
    torch.testing.assert_close(result, expected, rtol=0, atol=0)
    expected.float().square().mean().backward()
    result.float().square().mean().backward()
    assert tuple(actual.state_dict()) == keys
    assert all(a is b for a, b in zip(actual.parameters(), parameters, strict=True))
    for p, q in zip(actual.parameters(), reference.parameters(), strict=True):
        assert p.dtype == p.grad.dtype == torch.float32
        torch.testing.assert_close(p.grad, q.grad, rtol=0, atol=0)
    reference.eval()
    actual.eval()
    with torch.no_grad():
        torch.testing.assert_close(actual(tokens), reference(tokens), rtol=0, atol=0)


def test_mode_validation_and_noop():
    model = _model()
    forward = model.cores[0].ffn.forward
    apply_activation_execution(model, "baseline")
    assert model.cores[0].ffn.forward == forward
    with pytest.raises(ValueError, match="Unknown activation"):
        apply_activation_execution(model, "invalid")
    apply_activation_execution(model, "all")
    with pytest.raises(ValueError, match="already been applied"):
        apply_activation_execution(model, "all")
    with pytest.raises(ValueError, match="FP32 master"):
        apply_activation_execution(_model().bfloat16(), "all")


# CUDA tests require an explicit maintenance window. Normal CPU CI never starts
# CUDA or competes with the running training process merely on test collection.
cuda_window = pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1",
                                 reason="set WARPSTATE_TEST_CUDA=1 in an exclusive GPU window")


def _check_error(actual, expected, *, relative_rms=0.003, atol=0.0625):
    assert actual.dtype == expected.dtype
    assert torch.isfinite(actual).all()
    difference = actual.float() - expected.float()
    relative = difference.square().mean().sqrt() / expected.float().square().mean().sqrt().clamp_min(1e-20)
    assert relative.item() <= relative_rms, relative.item()
    assert difference.abs().max().item() <= atol, difference.abs().max().item()


@cuda_window
@pytest.mark.parametrize("shape", [(3, 224), (2, 32, 256), (257, 14336)])
@pytest.mark.parametrize("compiled", [False, True])
def test_cuda_swiglu_forward_and_backward(shape, compiled):
    torch.manual_seed(947)
    source = torch.randn(shape, device="cuda", dtype=torch.bfloat16)
    reference = source.clone().requires_grad_()
    actual = source.clone().requires_grad_()
    def reference_fn(x):
        up, gate = x.chunk(2, dim=-1)
        return F.silu(gate) * up
    fn = lambda x: bf16_swiglu(x, eager_rounding=not compiled)
    if compiled:
        reference_fn = torch.compile(reference_fn, fullgraph=True)
        fn = torch.compile(fn, fullgraph=True)
    expected = reference_fn(reference)
    result = fn(actual)
    grad = torch.randn_like(expected)
    expected.backward(grad)
    result.backward(grad)
    _check_error(result, expected, relative_rms=0.0002)
    _check_error(actual.grad, reference.grad, relative_rms=0.001)


@cuda_window
@pytest.mark.parametrize("shape", [(3, 24), (2, 16, 128), (257, 2048)])
@pytest.mark.parametrize("compiled", [False, True])
def test_cuda_rmsnorm_rounding_and_master_gradient(shape, compiled):
    torch.manual_seed(949)
    reference = RMSNorm(shape[-1]).cuda()
    with torch.no_grad():
        reference.weight.uniform_(0.1, 2.0)
    weight = reference.weight.detach().clone().requires_grad_()
    source = torch.randn(shape, device="cuda", dtype=torch.bfloat16)
    x_ref = source.clone().requires_grad_()
    x_actual = source.clone().requires_grad_()
    fn = lambda x, w: bf16_rmsnorm(x, w, reference.eps, eager_rounding=not compiled)[0]
    ref_fn = reference
    if compiled:
        ref_fn = torch.compile(reference, fullgraph=True)
        fn = torch.compile(fn, fullgraph=True)
    expected = ref_fn(x_ref)
    result = fn(x_actual, weight)
    # A strided upstream gradient verifies backward's contiguous handling.
    grad = torch.randn((*shape, 2), device="cuda", dtype=torch.bfloat16)[..., 0]
    expected.backward(grad)
    result.backward(grad)
    _check_error(result, expected, relative_rms=0.0002)
    _check_error(x_actual.grad, x_ref.grad, relative_rms=0.001)
    _check_error(weight.grad, reference.weight.grad,
                 relative_rms=0.003, atol=0.5)
    assert weight.grad.dtype == torch.float32


@cuda_window
def test_cuda_compiled_model_with_bf16_residuals_and_optimizer():
    from ops.residual_experiment import apply_bf16_residuals
    from run_40b import compile_training_cores
    reference = _model().cuda()
    actual = copy.deepcopy(reference)
    apply_bf16_residuals(reference)
    apply_bf16_residuals(actual)
    parameters = tuple(actual.parameters())
    keys = tuple(actual.state_dict())
    apply_activation_execution(actual, "all")
    compile_training_cores(reference)
    compile_training_cores(actual)
    optimizer = torch.optim.AdamW(actual.parameters(), lr=1e-4)
    tokens = torch.randint(48, (2, 8), device="cuda")
    with torch.autocast("cuda", dtype=torch.bfloat16):
        ref_loss = reference(tokens, tokens, return_logits=False)[1]
        loss = actual(tokens, tokens, return_logits=False)[1]
    ref_loss.backward()
    loss.backward()
    torch.testing.assert_close(loss, ref_loss, rtol=0.001, atol=0.001)
    grads = torch.cat([p.grad.flatten() for p in parameters])
    ref_grads = torch.cat([p.grad.flatten() for p in reference.parameters()])
    assert F.cosine_similarity(grads, ref_grads, dim=0).item() > 0.999
    assert (grads - ref_grads).norm().item() / ref_grads.norm().item() < 0.02
    optimizer.step()
    assert tuple(actual.state_dict()) == keys
    assert all(a is b for a, b in zip(actual.parameters(), parameters, strict=True))
    assert all(p.dtype == p.grad.dtype == optimizer.state[p]["exp_avg"].dtype
               == optimizer.state[p]["exp_avg_sq"].dtype == torch.float32 for p in parameters)
    restored = _model().cuda()
    restored.load_state_dict(actual.state_dict(), strict=True)
    actual.eval()
    with torch.no_grad():
        # Outside autocast the BF16 residual wrapper and activation patches both
        # retain exactly the archived model's original FP32 inference behavior.
        torch.testing.assert_close(actual(tokens), restored.eval()(tokens), rtol=0, atol=0)