File size: 13,775 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
"""Casting reuse must preserve rounding, FP32 leaves, and update boundaries."""

from contextlib import nullcontext
from datetime import timedelta
import json
import os
from pathlib import Path
import subprocess
import sys

import pytest
import torch
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel

from ops.residual_experiment import apply_bf16_residuals
from ops.weight_cast_execution import apply_weight_cast_cache
from warpstate import WarpStateConfig, WarpStateLM


def _model(*, dropout=0.0):
    torch.manual_seed(2718)
    cfg = WarpStateConfig(vocab_size=64, d_model=32, n_heads=4, n_cores=1,
                          logical_depth=1, ffn_hidden=64, chunk_size=4,
                          parallel_memory=True, gradient_checkpointing=False,
                          loss_chunk_size=8, dropout=dropout)
    model = WarpStateLM(cfg).train()
    apply_bf16_residuals(model)
    return model


def _batch(seed):
    generator = torch.Generator().manual_seed(seed)
    return (torch.randint(64, (1, 8), generator=generator),
            torch.randint(64, (1, 8), generator=generator))


@pytest.mark.parametrize("compiled", [False, True])
@pytest.mark.parametrize("set_to_none", [False, True])
def test_eight_backward_calls_and_optimizer_updates_match_exactly(compiled, set_to_none):
    reference = _model(dropout=0.1)
    experiment = _model(dropout=0.1)
    original_parameters = tuple(experiment.parameters())
    original_keys = tuple(experiment.state_dict())
    if compiled:
        for model in [reference, experiment]:
            for core in model.cores:
                core.forward = torch.compile(core.forward, backend="aot_eager",
                                             fullgraph=True, dynamic=False)
    cache = apply_weight_cast_cache(experiment)
    optimizers = [torch.optim.AdamW(model.parameters(), lr=0.0001, foreach=False)
                  for model in [reference, experiment]]
    for update in range(3):
        for optimizer in optimizers:
            optimizer.zero_grad(set_to_none=set_to_none)
        with cache.update():
            assert cache.cached_tensor_count == 4 and cache.cached_bytes > 0
            for microstep in range(8):
                inputs, targets = _batch(500 + update * 8 + microstep)
                torch.manual_seed(1000 + update * 8 + microstep)
                with torch.autocast("cpu", dtype=torch.bfloat16):
                    _, expected_loss = reference(inputs, targets, return_logits=False)
                (expected_loss / 8).backward()
                torch.manual_seed(1000 + update * 8 + microstep)
                with torch.autocast("cpu", dtype=torch.bfloat16):
                    _, actual_loss = experiment(inputs, targets, return_logits=False)
                (actual_loss / 8).backward()
                torch.testing.assert_close(actual_loss, expected_loss, rtol=0, atol=0)
                for actual, expected in zip(experiment.parameters(), reference.parameters()):
                    assert actual.dtype == actual.grad.dtype == torch.float32
                    torch.testing.assert_close(actual.grad, expected.grad, rtol=0, atol=0)
            assert tuple(experiment.state_dict()) == original_keys
            assert all(a is b for a, b in zip(original_parameters, experiment.parameters()))
        assert not cache.active and cache.cached_tensor_count == cache.cached_bytes == 0
        for optimizer in optimizers:
            optimizer.step()
        for actual, expected in zip(experiment.parameters(), reference.parameters()):
            torch.testing.assert_close(actual, expected, rtol=0, atol=0)
            state = optimizers[1].state[actual]
            assert state["exp_avg"].dtype == state["exp_avg_sq"].dtype == torch.float32


def test_checkpoint_load_between_scopes_uses_fresh_casts_and_preserves_fp32_state():
    model = _model()
    cache = apply_weight_cast_cache(model)
    with cache.update():
        old = cache._views["wrapped.cores.0.in_proj.weight"].detach().clone()
    checkpoint = {name: value.clone() for name, value in model.state_dict().items()}
    checkpoint["cores.0.in_proj.weight"].add_(0.5)
    model.load_state_dict(checkpoint, strict=True)
    with cache.update():
        current = cache._views["wrapped.cores.0.in_proj.weight"]
        assert not torch.equal(old, current)
        torch.testing.assert_close(current, model.cores[0].in_proj.weight.bfloat16(), rtol=0, atol=0)
    assert all(parameter.dtype == torch.float32 for parameter in model.parameters())


def test_scope_failure_restores_parameters_and_releases_casts():
    model = _model()
    masters = tuple(model.parameters())
    cache = apply_weight_cast_cache(model)
    inputs, targets = _batch(40)
    with pytest.raises(ValueError, match="must be divisible"):
        with cache.update(), torch.autocast("cpu", dtype=torch.bfloat16):
            model(inputs[:, :-1], targets[:, :-1])
    assert not cache.active and cache.cached_bytes == 0
    assert all(a is b for a, b in zip(masters, model.parameters()))
    with cache.update(), torch.autocast("cpu", dtype=torch.bfloat16):
        _, loss = model(inputs, targets, return_logits=False)
        loss.backward()


@pytest.mark.parametrize("mutation", ["in_place", "load_state", "replace"])
def test_parameter_mutation_inside_scope_is_detected_and_cleared(mutation):
    model = _model()
    cache = apply_weight_cast_cache(model)
    with pytest.raises(RuntimeError, match="changed"):
        with cache.update():
            if mutation == "in_place":
                with torch.no_grad():
                    model.cores[0].in_proj.weight.add_(0.01)
            elif mutation == "load_state":
                model.load_state_dict(model.state_dict())
            else:
                model.cores[0].in_proj.weight = torch.nn.Parameter(
                    model.cores[0].in_proj.weight.detach().clone())
    assert not cache.active and cache.cached_bytes == 0


def test_outside_scope_and_inference_are_unchanged_and_do_not_create_cache():
    reference = _model()
    model = _model()
    cache = apply_weight_cast_cache(model)
    inputs, _ = _batch(71)
    for candidate in [reference, model]:
        candidate.eval()
    with torch.inference_mode(), torch.autocast("cpu", dtype=torch.bfloat16):
        torch.testing.assert_close(model(inputs), reference(inputs), rtol=0, atol=0)
    assert not cache.active and cache.cached_bytes == 0
    with pytest.raises(RuntimeError, match="training with gradients"):
        with cache.update():
            pass


def test_incompatible_modes_and_nested_scopes_fail_explicitly():
    model = _model()
    model.cfg.logical_depth = 2
    with pytest.raises(ValueError, match="one use"):
        apply_weight_cast_cache(model)
    model.cfg.logical_depth = 1
    model.cfg.gradient_checkpointing = True
    with pytest.raises(ValueError, match="checkpointing off"):
        apply_weight_cast_cache(model)
    model.cfg.gradient_checkpointing = False
    cache = apply_weight_cast_cache(model)
    with pytest.raises(ValueError, match="already been applied"):
        apply_weight_cast_cache(model)
    with cache.update():
        with pytest.raises(RuntimeError, match="cannot be nested"):
            with cache.update():
                pass
        with pytest.raises(RuntimeError, match="requires BF16 autocast"):
            model(_batch(2)[0])


def _ddp_worker(rank, directory):
    torch.set_num_threads(1)
    directory = Path(directory)
    dist.init_process_group("gloo", init_method=(directory / "rendezvous").as_uri(),
                            rank=rank, world_size=2, timeout=timedelta(seconds=30))
    try:
        reference = DistributedDataParallel(_model(), gradient_as_bucket_view=True)
        raw = _model()
        cache = apply_weight_cast_cache(raw)
        experiment = DistributedDataParallel(raw, gradient_as_bucket_view=True)
        optimizers = [torch.optim.AdamW(model.parameters(), lr=0.0001, foreach=False)
                      for model in [reference, experiment]]
        for update in range(3):
            for optimizer in optimizers:
                optimizer.zero_grad(set_to_none=True)
            with cache.update():
                for microstep in range(8):
                    inputs, targets = _batch(100 + rank * 100 + update * 8 + microstep)
                    for model in [reference, experiment]:
                        with model.no_sync() if microstep < 7 else nullcontext():
                            with torch.autocast("cpu", dtype=torch.bfloat16):
                                _, loss = model(inputs, targets, return_logits=False)
                            (loss / 8).backward()
                for actual, expected in zip(experiment.parameters(), reference.parameters()):
                    assert actual.grad.dtype == expected.grad.dtype == torch.float32
                    torch.testing.assert_close(actual.grad, expected.grad, rtol=0, atol=0)
            assert not cache.active
            for optimizer in optimizers:
                optimizer.step()
            for actual, expected in zip(experiment.parameters(), reference.parameters()):
                torch.testing.assert_close(actual, expected, rtol=0, atol=0)
        (directory / f"rank-{rank}.json").write_text(json.dumps({"updates": 3, "equal": True}))
    finally:
        dist.destroy_process_group()


def test_two_rank_accumulation_preserves_ddp_leaf_hooks_exactly(tmp_path):
    environment = os.environ.copy()
    environment.update(CUDA_VISIBLE_DEVICES="", OMP_NUM_THREADS="1",
                       MKL_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1")
    processes = []
    try:
        for rank in range(2):
            processes.append(subprocess.Popen(
                [sys.executable, str(Path(__file__).resolve()), str(rank), str(tmp_path)],
                env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
            ))
        for rank, process in enumerate(processes):
            output, _ = process.communicate(timeout=90)
            assert process.returncode == 0, f"Rank {rank} failed:\n{output}"
    finally:
        for process in processes:
            if process.poll() is None:
                process.kill()
        for process in processes:
            process.wait(timeout=10)
    assert all(json.loads((tmp_path / f"rank-{rank}.json").read_text())["equal"]
               for rank in range(2))


@pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1",
                    reason="requires an explicitly reserved CUDA device")
def test_cuda_inductor_core_weight_cache_matches_bf16_updates(tmp_path):
    from ops.loss_experiment import apply_loss_execution

    torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False
    cfg = WarpStateConfig(vocab_size=256, d_model=256, n_heads=4, n_cores=2,
                          logical_depth=2, ffn_hidden=896, chunk_size=128,
                          parallel_memory=True, gradient_checkpointing=False,
                          loss_chunk_size=256)
    models = []
    for _ in range(2):
        torch.manual_seed(314)
        model = WarpStateLM(cfg).cuda().train()
        for core in model.cores:
            core.forward = torch.compile(core.forward, fullgraph=True, dynamic=False)
        apply_loss_execution(model, "compiled-full")
        apply_bf16_residuals(model)
        models.append(model)
    reference, experiment = models
    cache = apply_weight_cast_cache(experiment)
    optimizers = [torch.optim.AdamW(model.parameters(), lr=0.0001, fused=True)
                  for model in models]
    report = {"maximum_loss_difference": 0.0, "maximum_gradient_difference": 0.0,
              "maximum_parameter_difference": 0.0, "updates": 3, "microsteps": 8}
    for update in range(3):
        for optimizer in optimizers:
            optimizer.zero_grad(set_to_none=True)
        with cache.update():
            for microstep in range(8):
                generator = torch.Generator().manual_seed(42 + update * 8 + microstep)
                inputs = torch.randint(256, (1, 256), generator=generator).cuda()
                targets = torch.randint(256, (1, 256), generator=generator).cuda()
                losses = []
                for model in models:
                    with torch.autocast("cuda", dtype=torch.bfloat16):
                        _, loss = model(inputs, targets, return_logits=False)
                    (loss / 8).backward()
                    losses.append(loss.detach())
                report["maximum_loss_difference"] = max(report["maximum_loss_difference"],
                    (losses[0] - losses[1]).abs().item())
                for actual, expected in zip(experiment.parameters(), reference.parameters()):
                    report["maximum_gradient_difference"] = max(report["maximum_gradient_difference"],
                        (actual.grad - expected.grad).abs().max().item())
                    assert actual.grad.dtype == expected.grad.dtype == torch.float32
        for optimizer in optimizers:
            optimizer.step()
        for actual, expected in zip(experiment.parameters(), reference.parameters()):
            report["maximum_parameter_difference"] = max(report["maximum_parameter_difference"],
                (actual - expected).abs().max().item())
    (tmp_path / "cuda-weight-cast-comparison.json").write_text(json.dumps(report, indent=2) + "\n")
    print("CUDA_WEIGHT_CAST_COMPARISON " + json.dumps(report), flush=True)
    assert report["maximum_loss_difference"] == 0.0, report
    assert report["maximum_gradient_difference"] == 0.0, report
    assert report["maximum_parameter_difference"] == 0.0, report


if __name__ == "__main__":
    _ddp_worker(int(sys.argv[1]), sys.argv[2])