File size: 6,205 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
"""BF16 residual execution keeps FP32 masters and changes only compute rounding."""

from __future__ import annotations

import copy
import json
from pathlib import Path

import pytest
import torch
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel

from ops.residual_experiment import apply_bf16_residuals
from run_40b import compile_training_cores
from warpstate import WarpStateConfig, WarpStateLM


def _model():
    torch.manual_seed(685)
    config = WarpStateConfig(vocab_size=128, d_model=64, n_heads=4,
                             n_cores=1, logical_depth=2, ffn_hidden=112,
                             chunk_size=4, parallel_memory=True,
                             gradient_checkpointing=False, loss_chunk_size=8)
    return WarpStateLM(config).train()


@pytest.mark.parametrize("compile_cores", [False, True])
def test_bf16_residual_numerics_and_fp32_state(compile_cores, tmp_path, monkeypatch):
    # CPU eager backend checks Dynamo capture and functional parameter swapping;
    # production CUDA Inductor speed and numerics require separate GPU screening.
    if compile_cores:
        actual_compile = torch.compile
        monkeypatch.setattr(torch, "compile", lambda fn, **kw: actual_compile(fn, backend="eager", **kw))
    reference = _model()
    experiment = copy.deepcopy(reference)
    parameters = tuple(experiment.parameters())
    keys = tuple(experiment.state_dict())
    optimizer = torch.optim.AdamW(experiment.parameters(), lr=1e-4)
    apply_bf16_residuals(experiment)
    if compile_cores:
        compile_training_cores(reference)
        compile_training_cores(experiment)
    inputs = torch.randint(128, (2, 16))
    targets = torch.randint(128, inputs.shape)
    targets[:, -3:] = -100
    residual_dtypes = []
    hook = experiment.cores[0].register_forward_pre_hook(lambda _module, args: residual_dtypes.append(args[0].dtype))
    with torch.autocast("cpu", dtype=torch.bfloat16):
        _, ref_loss = reference(inputs, targets, return_logits=False)
        _, loss = experiment(inputs, targets, return_logits=False)
    hook.remove()
    assert residual_dtypes == [torch.bfloat16] * experiment.cfg.logical_depth
    ref_loss.backward()
    loss.backward()
    assert tuple(experiment.state_dict()) == keys
    assert all(a is b for a, b in zip(parameters, experiment.parameters(), strict=True))
    assert all(parameter.dtype == parameter.grad.dtype == torch.float32 for parameter in parameters)
    assert all(torch.isfinite(parameter.grad).all() for parameter in parameters)
    actual_gradients = torch.cat([parameter.grad.flatten() for parameter in parameters])
    reference_gradients = torch.cat([parameter.grad.flatten() for parameter in reference.parameters()])
    cosine = float(torch.nn.functional.cosine_similarity(actual_gradients, reference_gradients, dim=0))
    norm_ratio = float(actual_gradients.norm() / reference_gradients.norm())
    relative_loss_difference = float((loss.detach() - ref_loss.detach()).abs() / ref_loss.detach().abs())
    report = {
        "compiled_cores": compile_cores, "reference_loss": float(ref_loss.detach()),
        "loss": float(loss.detach()), "relative_loss_difference": relative_loss_difference,
        "gradient_cosine": cosine, "gradient_norm_ratio": norm_ratio,
    }
    print("BF16_RESIDUAL_NUMERICS " + json.dumps(report, sort_keys=True), flush=True)
    (tmp_path / "bf16-residual-numerics.json").write_text(json.dumps(report, indent=2) + "\n")
    assert relative_loss_difference <= 0.01, report
    assert cosine >= 0.99, report
    assert 0.95 <= norm_ratio <= 1.05, report
    optimizer.step()
    for parameter in parameters:
        assert parameter.dtype == torch.float32
        assert optimizer.state[parameter]["exp_avg"].dtype == torch.float32
        assert optimizer.state[parameter]["exp_avg_sq"].dtype == torch.float32
    restored = _model()
    restored.load_state_dict(experiment.state_dict(), strict=True)
    assert tuple(restored.state_dict()) == keys
    for name, expected in experiment.state_dict().items():
        torch.testing.assert_close(restored.state_dict()[name], expected, rtol=0, atol=0)


def test_bf16_residual_restores_parameters_on_error_and_preserves_non_autocast():
    model = _model()
    reference = copy.deepcopy(model)
    original_parameters = tuple(model.parameters())
    apply_bf16_residuals(model)
    inputs = torch.randint(128, (1, 16))
    torch.testing.assert_close(model(inputs), reference(inputs), rtol=0, atol=0)
    with pytest.raises(ValueError, match="must be divisible"):
        with torch.autocast("cpu", dtype=torch.bfloat16):
            model(inputs[:, :-1])
    assert all(a is b for a, b in zip(original_parameters, model.parameters(), strict=True))
    assert all(parameter.dtype == torch.float32 for parameter in model.parameters())
    with pytest.raises(ValueError, match="already been applied"):
        apply_bf16_residuals(model)


def test_bf16_residual_ddp_reducer_owns_fp32_master_gradients(tmp_path):
    if not dist.is_available() or not dist.is_gloo_available():
        pytest.skip("requires the CPU Gloo distributed backend")
    if dist.is_initialized():
        pytest.skip("requires ownership of an isolated default process group")
    assert tmp_path.resolve().is_relative_to(Path(__file__).resolve().parents[1])
    dist.init_process_group("gloo", init_method=(tmp_path / "ddp-init").as_uri(), rank=0, world_size=1)
    try:
        model = _model()
        masters = tuple(model.parameters())
        apply_bf16_residuals(model)
        distributed = DistributedDataParallel(model)
        inputs = torch.randint(128, (2, 16))
        for _ in range(2):  # Second forward detects unfinished reducer work.
            distributed.zero_grad(set_to_none=True)
            with torch.autocast("cpu", dtype=torch.bfloat16):
                _, loss = distributed(inputs, inputs, return_logits=False)
            loss.backward()
            assert all(parameter.grad is not None and parameter.grad.dtype == torch.float32 for parameter in masters)
        assert all(a is b for a, b in zip(masters, model.parameters(), strict=True))
    finally:
        dist.destroy_process_group()