Download test_weight_cast_execution.py from zerdovzad/WarpStateAI: direct link, hf CLI and curl.
- Browser
- Download file 13.8 kB
-
https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_weight_cast_execution.py
- Command line
-
hf download hf://zerdovzad/WarpStateAI/test_weight_cast_execution.py
-
curl -L -o test_weight_cast_execution.py https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_weight_cast_execution.py
13.8 kB
| """Casting reuse must preserve rounding, FP32 leaves, and update boundaries.""" | |
| from contextlib import nullcontext | |
| from datetime import timedelta | |
| import json | |
| import os | |
| from pathlib import Path | |
| import subprocess | |
| import sys | |
| import pytest | |
| import torch | |
| import torch.distributed as dist | |
| from torch.nn.parallel import DistributedDataParallel | |
| from ops.residual_experiment import apply_bf16_residuals | |
| from ops.weight_cast_execution import apply_weight_cast_cache | |
| from warpstate import WarpStateConfig, WarpStateLM | |
| def _model(*, dropout=0.0): | |
| torch.manual_seed(2718) | |
| cfg = WarpStateConfig(vocab_size=64, d_model=32, n_heads=4, n_cores=1, | |
| logical_depth=1, ffn_hidden=64, chunk_size=4, | |
| parallel_memory=True, gradient_checkpointing=False, | |
| loss_chunk_size=8, dropout=dropout) | |
| model = WarpStateLM(cfg).train() | |
| apply_bf16_residuals(model) | |
| return model | |
| def _batch(seed): | |
| generator = torch.Generator().manual_seed(seed) | |
| return (torch.randint(64, (1, 8), generator=generator), | |
| torch.randint(64, (1, 8), generator=generator)) | |
| def test_eight_backward_calls_and_optimizer_updates_match_exactly(compiled, set_to_none): | |
| reference = _model(dropout=0.1) | |
| experiment = _model(dropout=0.1) | |
| original_parameters = tuple(experiment.parameters()) | |
| original_keys = tuple(experiment.state_dict()) | |
| if compiled: | |
| for model in [reference, experiment]: | |
| for core in model.cores: | |
| core.forward = torch.compile(core.forward, backend="aot_eager", | |
| fullgraph=True, dynamic=False) | |
| cache = apply_weight_cast_cache(experiment) | |
| optimizers = [torch.optim.AdamW(model.parameters(), lr=0.0001, foreach=False) | |
| for model in [reference, experiment]] | |
| for update in range(3): | |
| for optimizer in optimizers: | |
| optimizer.zero_grad(set_to_none=set_to_none) | |
| with cache.update(): | |
| assert cache.cached_tensor_count == 4 and cache.cached_bytes > 0 | |
| for microstep in range(8): | |
| inputs, targets = _batch(500 + update * 8 + microstep) | |
| torch.manual_seed(1000 + update * 8 + microstep) | |
| with torch.autocast("cpu", dtype=torch.bfloat16): | |
| _, expected_loss = reference(inputs, targets, return_logits=False) | |
| (expected_loss / 8).backward() | |
| torch.manual_seed(1000 + update * 8 + microstep) | |
| with torch.autocast("cpu", dtype=torch.bfloat16): | |
| _, actual_loss = experiment(inputs, targets, return_logits=False) | |
| (actual_loss / 8).backward() | |
| torch.testing.assert_close(actual_loss, expected_loss, rtol=0, atol=0) | |
| for actual, expected in zip(experiment.parameters(), reference.parameters()): | |
| assert actual.dtype == actual.grad.dtype == torch.float32 | |
| torch.testing.assert_close(actual.grad, expected.grad, rtol=0, atol=0) | |
| assert tuple(experiment.state_dict()) == original_keys | |
| assert all(a is b for a, b in zip(original_parameters, experiment.parameters())) | |
| assert not cache.active and cache.cached_tensor_count == cache.cached_bytes == 0 | |
| for optimizer in optimizers: | |
| optimizer.step() | |
| for actual, expected in zip(experiment.parameters(), reference.parameters()): | |
| torch.testing.assert_close(actual, expected, rtol=0, atol=0) | |
| state = optimizers[1].state[actual] | |
| assert state["exp_avg"].dtype == state["exp_avg_sq"].dtype == torch.float32 | |
| def test_checkpoint_load_between_scopes_uses_fresh_casts_and_preserves_fp32_state(): | |
| model = _model() | |
| cache = apply_weight_cast_cache(model) | |
| with cache.update(): | |
| old = cache._views["wrapped.cores.0.in_proj.weight"].detach().clone() | |
| checkpoint = {name: value.clone() for name, value in model.state_dict().items()} | |
| checkpoint["cores.0.in_proj.weight"].add_(0.5) | |
| model.load_state_dict(checkpoint, strict=True) | |
| with cache.update(): | |
| current = cache._views["wrapped.cores.0.in_proj.weight"] | |
| assert not torch.equal(old, current) | |
| torch.testing.assert_close(current, model.cores[0].in_proj.weight.bfloat16(), rtol=0, atol=0) | |
| assert all(parameter.dtype == torch.float32 for parameter in model.parameters()) | |
| def test_scope_failure_restores_parameters_and_releases_casts(): | |
| model = _model() | |
| masters = tuple(model.parameters()) | |
| cache = apply_weight_cast_cache(model) | |
| inputs, targets = _batch(40) | |
| with pytest.raises(ValueError, match="must be divisible"): | |
| with cache.update(), torch.autocast("cpu", dtype=torch.bfloat16): | |
| model(inputs[:, :-1], targets[:, :-1]) | |
| assert not cache.active and cache.cached_bytes == 0 | |
| assert all(a is b for a, b in zip(masters, model.parameters())) | |
| with cache.update(), torch.autocast("cpu", dtype=torch.bfloat16): | |
| _, loss = model(inputs, targets, return_logits=False) | |
| loss.backward() | |
| def test_parameter_mutation_inside_scope_is_detected_and_cleared(mutation): | |
| model = _model() | |
| cache = apply_weight_cast_cache(model) | |
| with pytest.raises(RuntimeError, match="changed"): | |
| with cache.update(): | |
| if mutation == "in_place": | |
| with torch.no_grad(): | |
| model.cores[0].in_proj.weight.add_(0.01) | |
| elif mutation == "load_state": | |
| model.load_state_dict(model.state_dict()) | |
| else: | |
| model.cores[0].in_proj.weight = torch.nn.Parameter( | |
| model.cores[0].in_proj.weight.detach().clone()) | |
| assert not cache.active and cache.cached_bytes == 0 | |
| def test_outside_scope_and_inference_are_unchanged_and_do_not_create_cache(): | |
| reference = _model() | |
| model = _model() | |
| cache = apply_weight_cast_cache(model) | |
| inputs, _ = _batch(71) | |
| for candidate in [reference, model]: | |
| candidate.eval() | |
| with torch.inference_mode(), torch.autocast("cpu", dtype=torch.bfloat16): | |
| torch.testing.assert_close(model(inputs), reference(inputs), rtol=0, atol=0) | |
| assert not cache.active and cache.cached_bytes == 0 | |
| with pytest.raises(RuntimeError, match="training with gradients"): | |
| with cache.update(): | |
| pass | |
| def test_incompatible_modes_and_nested_scopes_fail_explicitly(): | |
| model = _model() | |
| model.cfg.logical_depth = 2 | |
| with pytest.raises(ValueError, match="one use"): | |
| apply_weight_cast_cache(model) | |
| model.cfg.logical_depth = 1 | |
| model.cfg.gradient_checkpointing = True | |
| with pytest.raises(ValueError, match="checkpointing off"): | |
| apply_weight_cast_cache(model) | |
| model.cfg.gradient_checkpointing = False | |
| cache = apply_weight_cast_cache(model) | |
| with pytest.raises(ValueError, match="already been applied"): | |
| apply_weight_cast_cache(model) | |
| with cache.update(): | |
| with pytest.raises(RuntimeError, match="cannot be nested"): | |
| with cache.update(): | |
| pass | |
| with pytest.raises(RuntimeError, match="requires BF16 autocast"): | |
| model(_batch(2)[0]) | |
| def _ddp_worker(rank, directory): | |
| torch.set_num_threads(1) | |
| directory = Path(directory) | |
| dist.init_process_group("gloo", init_method=(directory / "rendezvous").as_uri(), | |
| rank=rank, world_size=2, timeout=timedelta(seconds=30)) | |
| try: | |
| reference = DistributedDataParallel(_model(), gradient_as_bucket_view=True) | |
| raw = _model() | |
| cache = apply_weight_cast_cache(raw) | |
| experiment = DistributedDataParallel(raw, gradient_as_bucket_view=True) | |
| optimizers = [torch.optim.AdamW(model.parameters(), lr=0.0001, foreach=False) | |
| for model in [reference, experiment]] | |
| for update in range(3): | |
| for optimizer in optimizers: | |
| optimizer.zero_grad(set_to_none=True) | |
| with cache.update(): | |
| for microstep in range(8): | |
| inputs, targets = _batch(100 + rank * 100 + update * 8 + microstep) | |
| for model in [reference, experiment]: | |
| with model.no_sync() if microstep < 7 else nullcontext(): | |
| with torch.autocast("cpu", dtype=torch.bfloat16): | |
| _, loss = model(inputs, targets, return_logits=False) | |
| (loss / 8).backward() | |
| for actual, expected in zip(experiment.parameters(), reference.parameters()): | |
| assert actual.grad.dtype == expected.grad.dtype == torch.float32 | |
| torch.testing.assert_close(actual.grad, expected.grad, rtol=0, atol=0) | |
| assert not cache.active | |
| for optimizer in optimizers: | |
| optimizer.step() | |
| for actual, expected in zip(experiment.parameters(), reference.parameters()): | |
| torch.testing.assert_close(actual, expected, rtol=0, atol=0) | |
| (directory / f"rank-{rank}.json").write_text(json.dumps({"updates": 3, "equal": True})) | |
| finally: | |
| dist.destroy_process_group() | |
| def test_two_rank_accumulation_preserves_ddp_leaf_hooks_exactly(tmp_path): | |
| environment = os.environ.copy() | |
| environment.update(CUDA_VISIBLE_DEVICES="", OMP_NUM_THREADS="1", | |
| MKL_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1") | |
| processes = [] | |
| try: | |
| for rank in range(2): | |
| processes.append(subprocess.Popen( | |
| [sys.executable, str(Path(__file__).resolve()), str(rank), str(tmp_path)], | |
| env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, | |
| )) | |
| for rank, process in enumerate(processes): | |
| output, _ = process.communicate(timeout=90) | |
| assert process.returncode == 0, f"Rank {rank} failed:\n{output}" | |
| finally: | |
| for process in processes: | |
| if process.poll() is None: | |
| process.kill() | |
| for process in processes: | |
| process.wait(timeout=10) | |
| assert all(json.loads((tmp_path / f"rank-{rank}.json").read_text())["equal"] | |
| for rank in range(2)) | |
| def test_cuda_inductor_core_weight_cache_matches_bf16_updates(tmp_path): | |
| from ops.loss_experiment import apply_loss_execution | |
| torch.backends.cuda.matmul.allow_bf16_reduced_precision_reduction = False | |
| cfg = WarpStateConfig(vocab_size=256, d_model=256, n_heads=4, n_cores=2, | |
| logical_depth=2, ffn_hidden=896, chunk_size=128, | |
| parallel_memory=True, gradient_checkpointing=False, | |
| loss_chunk_size=256) | |
| models = [] | |
| for _ in range(2): | |
| torch.manual_seed(314) | |
| model = WarpStateLM(cfg).cuda().train() | |
| for core in model.cores: | |
| core.forward = torch.compile(core.forward, fullgraph=True, dynamic=False) | |
| apply_loss_execution(model, "compiled-full") | |
| apply_bf16_residuals(model) | |
| models.append(model) | |
| reference, experiment = models | |
| cache = apply_weight_cast_cache(experiment) | |
| optimizers = [torch.optim.AdamW(model.parameters(), lr=0.0001, fused=True) | |
| for model in models] | |
| report = {"maximum_loss_difference": 0.0, "maximum_gradient_difference": 0.0, | |
| "maximum_parameter_difference": 0.0, "updates": 3, "microsteps": 8} | |
| for update in range(3): | |
| for optimizer in optimizers: | |
| optimizer.zero_grad(set_to_none=True) | |
| with cache.update(): | |
| for microstep in range(8): | |
| generator = torch.Generator().manual_seed(42 + update * 8 + microstep) | |
| inputs = torch.randint(256, (1, 256), generator=generator).cuda() | |
| targets = torch.randint(256, (1, 256), generator=generator).cuda() | |
| losses = [] | |
| for model in models: | |
| with torch.autocast("cuda", dtype=torch.bfloat16): | |
| _, loss = model(inputs, targets, return_logits=False) | |
| (loss / 8).backward() | |
| losses.append(loss.detach()) | |
| report["maximum_loss_difference"] = max(report["maximum_loss_difference"], | |
| (losses[0] - losses[1]).abs().item()) | |
| for actual, expected in zip(experiment.parameters(), reference.parameters()): | |
| report["maximum_gradient_difference"] = max(report["maximum_gradient_difference"], | |
| (actual.grad - expected.grad).abs().max().item()) | |
| assert actual.grad.dtype == expected.grad.dtype == torch.float32 | |
| for optimizer in optimizers: | |
| optimizer.step() | |
| for actual, expected in zip(experiment.parameters(), reference.parameters()): | |
| report["maximum_parameter_difference"] = max(report["maximum_parameter_difference"], | |
| (actual - expected).abs().max().item()) | |
| (tmp_path / "cuda-weight-cast-comparison.json").write_text(json.dumps(report, indent=2) + "\n") | |
| print("CUDA_WEIGHT_CAST_COMPARISON " + json.dumps(report), flush=True) | |
| assert report["maximum_loss_difference"] == 0.0, report | |
| assert report["maximum_gradient_difference"] == 0.0, report | |
| assert report["maximum_parameter_difference"] == 0.0, report | |
| if __name__ == "__main__": | |
| _ddp_worker(int(sys.argv[1]), sys.argv[2]) | |