File size: 8,298 Bytes
c8ba9c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 | """Two-GPU full-state continuation checks for optional training optimizations."""
import json
import math
import pytest
import torch
from test_training_execution import (
TWO_GPUS,
assert_tensors_equal,
fixture_runs,
run_training,
)
CASES = [
pytest.param(
["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule"],
{"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
"overlap_producer_schedule": True, "overlap_swap_collectives": False,
"overlap_runtime_estimates": False, "overlap_memory_budget": 0.0,
"ddp_bucket_cap_mb": 128},
id="producer-scheduled-overlap",
),
pytest.param(
["--gradient-mode", "compiled-autograd-overlap", "--overlap-swap-collectives",
"--overlap-runtime-estimates"],
{"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
"overlap_swap_collectives": True, "overlap_runtime_estimates": True,
"overlap_memory_budget": 0.0, "ddp_bucket_cap_mb": 128},
id="runtime-estimated-overlap",
),
pytest.param(
["--gradient-mode", "compiled-autograd-overlap", "--overlap-swap-collectives",
"--overlap-memory-budget", "0.02", "--ddp-bucket-cap-mb", "64"],
{"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
"overlap_swap_collectives": True, "overlap_runtime_estimates": False,
"overlap_memory_budget": 0.02, "ddp_bucket_cap_mb": 64},
id="waits-past-collectives",
),
pytest.param(
["--gradient-mode", "compiled-autograd-overlap", "--matmul-plan", "__fixture_plan__"],
{"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
"cache_core_weight_casts": False, "ddp_batched_grad_copy": False, "ddp_bucket_cap_mb": 128},
id="matmul-epilogue-with-communication-overlap",
),
pytest.param(
["--gradient-mode", "eager", "--activation-mode", "all",
"--cache-core-weight-casts", "--ddp-batched-grad-copy",
"--ddp-bucket-cap-mb", "64"],
{"gradient_mode": "eager", "activation_mode": "all",
"cache_core_weight_casts": True, "ddp_batched_grad_copy": True,
"ddp_bucket_cap_mb": 64},
id="activations-cached-casts-batched-copy",
),
pytest.param(
["--gradient-mode", "compiled-autograd-overlap", "--activation-mode", "baseline",
"--no-cache-core-weight-casts", "--no-ddp-batched-grad-copy"],
{"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
"cache_core_weight_casts": False, "ddp_batched_grad_copy": False,
"ddp_bucket_cap_mb": 128},
id="compiled-autograd-overlap",
),
]
@TWO_GPUS
@pytest.mark.parametrize(("flags", "selected"), CASES)
def test_optimized_two_gpu_async_resume_preserves_full_state(tmp_path, flags, selected):
(continuous, resumed), common = fixture_runs(tmp_path, ["continuous", "resumed"])
common += [
"--loss-mode", "compiled-full", "--compile-core", "--compile-mode", "default",
"--bf16-residuals", "--no-activation-checkpointing", "--autocast-cache",
"--eval-every", "2", "--save-every", "2", *flags,
]
selected = dict(selected)
if "__fixture_plan__" in common:
from ops.matmul_execution import load_matmul_plan
tile = {"block_m": 16, "block_n": 32, "block_k": 16, "num_warps": 4, "num_stages": 2}
document = {"format": "warpstate-matmul-plan-v1", "entries": [
{"kind": "swiglu", "m": 16, "n": 64, "k": 16,
"a_stride": [16, 1], "b_stride": [1, 16], "dtype": "bfloat16",
"algorithm": "persistent", "tile": tile},
{"kind": "mm", "m": 64, "n": 16, "k": 16,
"a_stride": [1, 64], "b_stride": [16, 1], "dtype": "bfloat16",
"algorithm": "persistent_tma", "tile": tile},
]}
recipe = tmp_path / "matmul-plan.json"
recipe.write_text(json.dumps(document))
common[common.index("__fixture_plan__")] = str(recipe)
selected["matmul_execution"] = load_matmul_plan(recipe).metadata
initial = torch.load(tmp_path / "initial.pt", map_location="cpu", weights_only=False)
# The 73-token fixture consumes 32, 32, then 9 tokens. On its final update,
# rank zero consumes the remaining tokens and rank one has entirely masked
# targets. Resume must preserve that weighting and every rank's RNG state.
expected = run_training(continuous, common, 2, timeout=300,
expect_validation_display=True)
async_common = common + ["--async-checkpoint"]
partial = run_training(resumed, async_common, 2, ["--max-steps", "1"],
timeout=300, expect_async=True, expect_validation_display=True)
assert partial["tokens_seen"] == 32 and partial["step"] == 1
assert partial["val_step"] == 1 and partial["val_tokens"] == 32
actual = run_training(resumed, async_common, 2, timeout=300, expect_async=True,
expect_validation_display=True)
assert actual["format"] == expected["format"] == "warpstate-server-v1"
assert actual["step"] == expected["step"] == 3
assert actual["tokens_seen"] == expected["tokens_seen"] == 73
assert actual["origin"] == partial["origin"] == expected["origin"]
assert actual["execution_history"] == partial["execution_history"] == expected["execution_history"]
assert actual["signature"] == partial["signature"] == expected["signature"]
settings = actual["signature"]["settings"]
for name, value in selected.items():
assert settings[name] == value
assert settings["world_size"] == 2
assert settings["compile_core"] is True
assert settings["compile_mode"] == "default"
assert settings["loss_mode"] == "compiled-full"
assert settings["bf16_residuals"] is True
assert settings["activation_checkpointing"] is False
assert settings["seq_len"] * settings["micro_batch"] * settings["grad_accum"] * settings["world_size"] == 32
assert len(actual["rng_by_rank"]) == len(partial["rng_by_rank"]) == 2
for name in ("model", "optimizer", "rng_by_rank"):
assert_tensors_equal(actual[name], expected[name])
assert actual["model"].keys() == initial["model"].keys()
assert all(not name.startswith(("module.", "_orig_mod.")) for name in actual["model"])
assert all(value.dtype == torch.float32 and torch.isfinite(value).all()
for value in actual["model"].values())
assert actual["optimizer"]["state"]
for state in actual["optimizer"]["state"].values():
assert state["step"].item() == 3
for name in ("exp_avg", "exp_avg_sq"):
assert state[name].dtype == torch.float32 and torch.isfinite(state[name]).all()
assert actual["val_loss"] == expected["val_loss"]
assert math.isfinite(actual["val_loss"])
assert actual["val_step"] == expected["val_step"] == 3
assert actual["val_tokens"] == expected["val_tokens"] == 73
for directory, async_enabled in [(continuous, False), (resumed, True)]:
runtime = json.loads((directory / "run/settings.json").read_text())["runtime"]
for name, value in selected.items():
assert runtime[name] == value
assert runtime["async_checkpoint"] is async_enabled
assert runtime["bf16_residuals"] is True
assert runtime["eval_every"] == runtime["save_every"] == 2
assert not (directory / "run/latest.pt.tmp").exists()
records = [json.loads(line) for line in
(directory / "run/metrics.jsonl").read_text().splitlines()]
assert [record["step"] for record in records] == [1, 2, 3]
assert [record["tokens"] for record in records] == [32, 64, 73]
assert [record["lr"] for record in records] == [0.001, 0.002, 0.002]
assert all(math.isfinite(record["loss"]) and math.isfinite(record["grad_norm"])
for record in records)
validations = [json.loads(line) for line in
(resumed / "run/validation.jsonl").read_text().splitlines()]
assert [record["tokens"] for record in validations] == [32, 64, 73]
assert all(math.isfinite(record["loss"]) for record in validations)
|