WarpStateAI / test_residual_execution.py
zerdovzad's picture
Upload 139 files
c8ba9c6 verified
Raw History Blame Contribute Delete
6.21 kB
"""BF16 residual execution keeps FP32 masters and changes only compute rounding."""
from __future__ import annotations
import copy
import json
from pathlib import Path
import pytest
import torch
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel
from ops.residual_experiment import apply_bf16_residuals
from run_40b import compile_training_cores
from warpstate import WarpStateConfig, WarpStateLM
def _model():
torch.manual_seed(685)
config = WarpStateConfig(vocab_size=128, d_model=64, n_heads=4,
n_cores=1, logical_depth=2, ffn_hidden=112,
chunk_size=4, parallel_memory=True,
gradient_checkpointing=False, loss_chunk_size=8)
return WarpStateLM(config).train()
@pytest.mark.parametrize("compile_cores", [False, True])
def test_bf16_residual_numerics_and_fp32_state(compile_cores, tmp_path, monkeypatch):
# CPU eager backend checks Dynamo capture and functional parameter swapping;
# production CUDA Inductor speed and numerics require separate GPU screening.
if compile_cores:
actual_compile = torch.compile
monkeypatch.setattr(torch, "compile", lambda fn, **kw: actual_compile(fn, backend="eager", **kw))
reference = _model()
experiment = copy.deepcopy(reference)
parameters = tuple(experiment.parameters())
keys = tuple(experiment.state_dict())
optimizer = torch.optim.AdamW(experiment.parameters(), lr=1e-4)
apply_bf16_residuals(experiment)
if compile_cores:
compile_training_cores(reference)
compile_training_cores(experiment)
inputs = torch.randint(128, (2, 16))
targets = torch.randint(128, inputs.shape)
targets[:, -3:] = -100
residual_dtypes = []
hook = experiment.cores[0].register_forward_pre_hook(lambda _module, args: residual_dtypes.append(args[0].dtype))
with torch.autocast("cpu", dtype=torch.bfloat16):
_, ref_loss = reference(inputs, targets, return_logits=False)
_, loss = experiment(inputs, targets, return_logits=False)
hook.remove()
assert residual_dtypes == [torch.bfloat16] * experiment.cfg.logical_depth
ref_loss.backward()
loss.backward()
assert tuple(experiment.state_dict()) == keys
assert all(a is b for a, b in zip(parameters, experiment.parameters(), strict=True))
assert all(parameter.dtype == parameter.grad.dtype == torch.float32 for parameter in parameters)
assert all(torch.isfinite(parameter.grad).all() for parameter in parameters)
actual_gradients = torch.cat([parameter.grad.flatten() for parameter in parameters])
reference_gradients = torch.cat([parameter.grad.flatten() for parameter in reference.parameters()])
cosine = float(torch.nn.functional.cosine_similarity(actual_gradients, reference_gradients, dim=0))
norm_ratio = float(actual_gradients.norm() / reference_gradients.norm())
relative_loss_difference = float((loss.detach() - ref_loss.detach()).abs() / ref_loss.detach().abs())
report = {
"compiled_cores": compile_cores, "reference_loss": float(ref_loss.detach()),
"loss": float(loss.detach()), "relative_loss_difference": relative_loss_difference,
"gradient_cosine": cosine, "gradient_norm_ratio": norm_ratio,
}
print("BF16_RESIDUAL_NUMERICS " + json.dumps(report, sort_keys=True), flush=True)
(tmp_path / "bf16-residual-numerics.json").write_text(json.dumps(report, indent=2) + "\n")
assert relative_loss_difference <= 0.01, report
assert cosine >= 0.99, report
assert 0.95 <= norm_ratio <= 1.05, report
optimizer.step()
for parameter in parameters:
assert parameter.dtype == torch.float32
assert optimizer.state[parameter]["exp_avg"].dtype == torch.float32
assert optimizer.state[parameter]["exp_avg_sq"].dtype == torch.float32
restored = _model()
restored.load_state_dict(experiment.state_dict(), strict=True)
assert tuple(restored.state_dict()) == keys
for name, expected in experiment.state_dict().items():
torch.testing.assert_close(restored.state_dict()[name], expected, rtol=0, atol=0)
def test_bf16_residual_restores_parameters_on_error_and_preserves_non_autocast():
model = _model()
reference = copy.deepcopy(model)
original_parameters = tuple(model.parameters())
apply_bf16_residuals(model)
inputs = torch.randint(128, (1, 16))
torch.testing.assert_close(model(inputs), reference(inputs), rtol=0, atol=0)
with pytest.raises(ValueError, match="must be divisible"):
with torch.autocast("cpu", dtype=torch.bfloat16):
model(inputs[:, :-1])
assert all(a is b for a, b in zip(original_parameters, model.parameters(), strict=True))
assert all(parameter.dtype == torch.float32 for parameter in model.parameters())
with pytest.raises(ValueError, match="already been applied"):
apply_bf16_residuals(model)
def test_bf16_residual_ddp_reducer_owns_fp32_master_gradients(tmp_path):
if not dist.is_available() or not dist.is_gloo_available():
pytest.skip("requires the CPU Gloo distributed backend")
if dist.is_initialized():
pytest.skip("requires ownership of an isolated default process group")
assert tmp_path.resolve().is_relative_to(Path(__file__).resolve().parents[1])
dist.init_process_group("gloo", init_method=(tmp_path / "ddp-init").as_uri(), rank=0, world_size=1)
try:
model = _model()
masters = tuple(model.parameters())
apply_bf16_residuals(model)
distributed = DistributedDataParallel(model)
inputs = torch.randint(128, (2, 16))
for _ in range(2): # Second forward detects unfinished reducer work.
distributed.zero_grad(set_to_none=True)
with torch.autocast("cpu", dtype=torch.bfloat16):
_, loss = distributed(inputs, inputs, return_logits=False)
loss.backward()
assert all(parameter.grad is not None and parameter.grad.dtype == torch.float32 for parameter in masters)
assert all(a is b for a, b in zip(masters, model.parameters(), strict=True))
finally:
dist.destroy_process_group()