File size: 13,775 Bytes
c8ba9c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 | """Casting reuse must preserve rounding, FP32 leaves, and update boundaries."""
from contextlib import nullcontext
from datetime import timedelta
import json
import os
from pathlib import Path
import subprocess
import sys
import pytest
import torch
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel
from ops.residual_experiment import apply_bf16_residuals
from ops.weight_cast_execution import apply_weight_cast_cache
from warpstate import WarpStateConfig, WarpStateLM
def _model(*, dropout=0.0):
torch.manual_seed(2718)
cfg = WarpStateConfig(vocab_size=64, d_model=32, n_heads=4, n_cores=1,
logical_depth=1, ffn_hidden=64, chunk_size=4,
parallel_memory=True, gradient_checkpointing=False,
loss_chunk_size=8, dropout=dropout)
model = WarpStateLM(cfg).train()
apply_bf16_residuals(model)
return model
def _batch(seed):
generator = torch.Generator().manual_seed(seed)
return (torch.randint(64, (1, 8), generator=generator),
torch.randint(64, (1, 8), generator=generator))
@pytest.mark.parametrize("compiled", [False, True])
@pytest.mark.parametrize("set_to_none", [False, True])
def test_eight_backward_calls_and_optimizer_updates_match_exactly(compiled, set_to_none):
reference = _model(dropout=0.1)
experiment = _model(dropout=0.1)
original_parameters = tuple(experiment.parameters())
original_keys = tuple(experiment.state_dict())
if compiled:
for model in [reference, experiment]:
for core in model.cores:
core.forward = torch.compile(core.forward, backend="aot_eager",
fullgraph=True, dynamic=False)
cache = apply_weight_cast_cache(experiment)
optimizers = [torch.optim.AdamW(model.parameters(), lr=0.0001, foreach=False)
for model in [reference, experiment]]
for update in range(3):
for optimizer in optimizers:
optimizer.zero_grad(set_to_none=set_to_none)
with cache.update():
assert cache.cached_tensor_count == 4 and cache.cached_bytes > 0
for microstep in range(8):
inputs, targets = _batch(500 + update * 8 + microstep)
torch.manual_seed(1000 + update * 8 + microstep)
with torch.autocast("cpu", dtype=torch.bfloat16):
_, expected_loss = reference(inputs, targets, return_logits=False)
(expected_loss / 8).backward()
torch.manual_seed(1000 + update * 8 + microstep)
with torch.autocast("cpu", dtype=torch.bfloat16):
_, actual_loss = experiment(inputs, targets, return_logits=False)
(actual_loss / 8).backward()
torch.testing.assert_close(actual_loss, expected_loss, rtol=0, atol=0)
for actual, expected in zip(experiment.parameters(), reference.parameters()):
assert actual.dtype == actual.grad.dtype == torch.float32
torch.testing.assert_close(actual.grad, expected.grad, rtol=0, atol=0)
assert tuple(experiment.state_dict()) == original_keys
assert all(a is b for a, b in zip(original_parameters, experiment.parameters()))
assert not cache.active and cache.cached_tensor_count == cache.cached_bytes == 0
for optimizer in optimizers:
optimizer.step()
for actual, expected in zip(experiment.parameters(), reference.parameters()):
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
state = optimizers[1].state[actual]
assert state["exp_avg"].dtype == state["exp_avg_sq"].dtype == torch.float32
def test_checkpoint_load_between_scopes_uses_fresh_casts_and_preserves_fp32_state():
model = _model()
cache = apply_weight_cast_cache(model)
with cache.update():
old = cache._views["wrapped.cores.0.in_proj.weight"].detach().clone()
checkpoint = {name: value.clone() for name, value in model.state_dict().items()}
checkpoint["cores.0.in_proj.weight"].add_(0.5)
model.load_state_dict(checkpoint, strict=True)
with cache.update():
current = cache._views["wrapped.cores.0.in_proj.weight"]
assert not torch.equal(old, current)
torch.testing.assert_close(current, model.cores[0].in_proj.weight.bfloat16(), rtol=0, atol=0)
assert all(parameter.dtype == torch.float32 for parameter in model.parameters())
def test_scope_failure_restores_parameters_and_releases_casts():
model = _model()
masters = tuple(model.parameters())
cache = apply_weight_cast_cache(model)
inputs, targets = _batch(40)
with pytest.raises(ValueError, match="must be divisible"):
with cache.update(), torch.autocast("cpu", dtype=torch.bfloat16):
model(inputs[:, :-1], targets[:, :-1])
assert not cache.active and cache.cached_bytes == 0
assert all(a is b for a, b in zip(masters, model.parameters()))
with cache.update(), torch.autocast("cpu", dtype=torch.bfloat16):
_, loss = model(inputs, targets, return_logits=False)
loss.backward()
@pytest.mark.parametrize("mutation", ["in_place", "load_state", "replace"])
def test_parameter_mutation_inside_scope_is_detected_and_cleared(mutation):
model = _model()
cache = apply_weight_cast_cache(model)
with pytest.raises(RuntimeError, match="changed"):
with cache.update():
if mutation == "in_place":
with torch.no_grad():
model.cores[0].in_proj.weight.add_(0.01)
elif mutation == "load_state":
model.load_state_dict(model.state_dict())
else:
model.cores[0].in_proj.weight = torch.nn.Parameter(
model.cores[0].in_proj.weight.detach().clone())
assert not cache.active and cache.cached_bytes == 0
def test_outside_scope_and_inference_are_unchanged_and_do_not_create_cache():
reference = _model()
model = _model()
cache = apply_weight_cast_cache(model)
inputs, _ = _batch(71)
for candidate in [reference, model]:
candidate.eval()
with torch.inference_mode(), torch.autocast("cpu", dtype=torch.bfloat16):
torch.testing.assert_close(model(inputs), reference(inputs), rtol=0, atol=0)
assert not cache.active and cache.cached_bytes == 0
with pytest.raises(RuntimeError, match="training with gradients"):
with cache.update():
pass
def test_incompatible_modes_and_nested_scopes_fail_explicitly():
model = _model()
model.cfg.logical_depth = 2
with pytest.raises(ValueError, match="one use"):
apply_weight_cast_cache(model)
model.cfg.logical_depth = 1
model.cfg.gradient_checkpointing = True
with pytest.raises(ValueError, match="checkpointing off"):
apply_weight_cast_cache(model)
model.cfg.gradient_checkpointing = False
cache = apply_weight_cast_cache(model)
with pytest.raises(ValueError, match="already been applied"):
apply_weight_cast_cache(model)
with cache.update():
with pytest.raises(RuntimeError, match="cannot be nested"):
with cache.update():
pass
with pytest.raises(RuntimeError, match="requires BF16 autocast"):
model(_batch(2)[0])
def _ddp_worker(rank, directory):
torch.set_num_threads(1)
directory = Path(directory)
dist.init_process_group("gloo", init_method=(directory / "rendezvous").as_uri(),
rank=rank, world_size=2, timeout=timedelta(seconds=30))
try:
reference = DistributedDataParallel(_model(), gradient_as_bucket_view=True)
raw = _model()
cache = apply_weight_cast_cache(raw)
experiment = DistributedDataParallel(raw, gradient_as_bucket_view=True)
optimizers = [torch.optim.AdamW(model.parameters(), lr=0.0001, foreach=False)
for model in [reference, experiment]]
for update in range(3):
for optimizer in optimizers:
optimizer.zero_grad(set_to_none=True)
with cache.update():
for microstep in range(8):
inputs, targets = _batch(100 + rank * 100 + update * 8 + microstep)
for model in [reference, experiment]:
with model.no_sync() if microstep < 7 else nullcontext():
with torch.autocast("cpu", dtype=torch.bfloat16):
_, loss = model(inputs, targets, return_logits=False)
(loss / 8).backward()
for actual, expected in zip(experiment.parameters(), reference.parameters()):
assert actual.grad.dtype == expected.grad.dtype == torch.float32
torch.testing.assert_close(actual.grad, expected.grad, rtol=0, atol=0)
assert not cache.active
for optimizer in optimizers:
optimizer.step()
for actual, expected in zip(experiment.parameters(), reference.parameters()):
torch.testing.assert_close(actual, expected, rtol=0, atol=0)
(directory / f"rank-{rank}.json").write_text(json.dumps({"updates": 3, "equal": True}))
finally:
dist.destroy_process_group()
def test_two_rank_accumulation_preserves_ddp_leaf_hooks_exactly(tmp_path):
environment = os.environ.copy()
environment.update(CUDA_VISIBLE_DEVICES="", OMP_NUM_THREADS="1",
MKL_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1")
processes = []
try:
for rank in range(2):
processes.append(subprocess.Popen(
[sys.executable, str(Path(__file__).resolve()), str(rank), str(tmp_path)],
env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
))
for rank, process in enumerate(processes):
output, _ = process.communicate(timeout=90)
assert process.returncode == 0, f"Rank {rank} failed:\n{output}"
finally:
for process in processes:
if process.poll() is None:
process.kill()
for process in processes:
process.wait(timeout=10)
assert all(json.loads((tmp_path / f"rank-{rank}.json").read_text())["equal"]
for rank in range(2))
@pytest.mark.skipif(os.environ.get("WARPSTATE_TEST_CUDA") != "1",
reason="requires an explicitly reserved CUDA device")
def test_cuda_inductor_core_weight_cache_matches_bf16_updates(tmp_path):
from ops.loss_experiment import apply_loss_execution
torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False
cfg = WarpStateConfig(vocab_size=256, d_model=256, n_heads=4, n_cores=2,
logical_depth=2, ffn_hidden=896, chunk_size=128,
parallel_memory=True, gradient_checkpointing=False,
loss_chunk_size=256)
models = []
for _ in range(2):
torch.manual_seed(314)
model = WarpStateLM(cfg).cuda().train()
for core in model.cores:
core.forward = torch.compile(core.forward, fullgraph=True, dynamic=False)
apply_loss_execution(model, "compiled-full")
apply_bf16_residuals(model)
models.append(model)
reference, experiment = models
cache = apply_weight_cast_cache(experiment)
optimizers = [torch.optim.AdamW(model.parameters(), lr=0.0001, fused=True)
for model in models]
report = {"maximum_loss_difference": 0.0, "maximum_gradient_difference": 0.0,
"maximum_parameter_difference": 0.0, "updates": 3, "microsteps": 8}
for update in range(3):
for optimizer in optimizers:
optimizer.zero_grad(set_to_none=True)
with cache.update():
for microstep in range(8):
generator = torch.Generator().manual_seed(42 + update * 8 + microstep)
inputs = torch.randint(256, (1, 256), generator=generator).cuda()
targets = torch.randint(256, (1, 256), generator=generator).cuda()
losses = []
for model in models:
with torch.autocast("cuda", dtype=torch.bfloat16):
_, loss = model(inputs, targets, return_logits=False)
(loss / 8).backward()
losses.append(loss.detach())
report["maximum_loss_difference"] = max(report["maximum_loss_difference"],
(losses[0] - losses[1]).abs().item())
for actual, expected in zip(experiment.parameters(), reference.parameters()):
report["maximum_gradient_difference"] = max(report["maximum_gradient_difference"],
(actual.grad - expected.grad).abs().max().item())
assert actual.grad.dtype == expected.grad.dtype == torch.float32
for optimizer in optimizers:
optimizer.step()
for actual, expected in zip(experiment.parameters(), reference.parameters()):
report["maximum_parameter_difference"] = max(report["maximum_parameter_difference"],
(actual - expected).abs().max().item())
(tmp_path / "cuda-weight-cast-comparison.json").write_text(json.dumps(report, indent=2) + "\n")
print("CUDA_WEIGHT_CAST_COMPARISON " + json.dumps(report), flush=True)
assert report["maximum_loss_difference"] == 0.0, report
assert report["maximum_gradient_difference"] == 0.0, report
assert report["maximum_parameter_difference"] == 0.0, report
if __name__ == "__main__":
_ddp_worker(int(sys.argv[1]), sys.argv[2])
|