File size: 8,298 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
"""Two-GPU full-state continuation checks for optional training optimizations."""

import json
import math

import pytest
import torch

from test_training_execution import (
    TWO_GPUS,
    assert_tensors_equal,
    fixture_runs,
    run_training,
)


CASES = [
    pytest.param(
        ["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule"],
        {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
         "overlap_producer_schedule": True, "overlap_swap_collectives": False,
         "overlap_runtime_estimates": False, "overlap_memory_budget": 0.0,
         "ddp_bucket_cap_mb": 128},
        id="producer-scheduled-overlap",
    ),
    pytest.param(
        ["--gradient-mode", "compiled-autograd-overlap", "--overlap-swap-collectives",
         "--overlap-runtime-estimates"],
        {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
         "overlap_swap_collectives": True, "overlap_runtime_estimates": True,
         "overlap_memory_budget": 0.0, "ddp_bucket_cap_mb": 128},
        id="runtime-estimated-overlap",
    ),
    pytest.param(
        ["--gradient-mode", "compiled-autograd-overlap", "--overlap-swap-collectives",
         "--overlap-memory-budget", "0.02", "--ddp-bucket-cap-mb", "64"],
        {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
         "overlap_swap_collectives": True, "overlap_runtime_estimates": False,
         "overlap_memory_budget": 0.02, "ddp_bucket_cap_mb": 64},
        id="waits-past-collectives",
    ),
    pytest.param(
        ["--gradient-mode", "compiled-autograd-overlap", "--matmul-plan", "__fixture_plan__"],
        {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
         "cache_core_weight_casts": False, "ddp_batched_grad_copy": False, "ddp_bucket_cap_mb": 128},
        id="matmul-epilogue-with-communication-overlap",
    ),
    pytest.param(
        ["--gradient-mode", "eager", "--activation-mode", "all",
         "--cache-core-weight-casts", "--ddp-batched-grad-copy",
         "--ddp-bucket-cap-mb", "64"],
        {"gradient_mode": "eager", "activation_mode": "all",
         "cache_core_weight_casts": True, "ddp_batched_grad_copy": True,
         "ddp_bucket_cap_mb": 64},
        id="activations-cached-casts-batched-copy",
    ),
    pytest.param(
        ["--gradient-mode", "compiled-autograd-overlap", "--activation-mode", "baseline",
         "--no-cache-core-weight-casts", "--no-ddp-batched-grad-copy"],
        {"gradient_mode": "compiled-autograd-overlap", "activation_mode": "baseline",
         "cache_core_weight_casts": False, "ddp_batched_grad_copy": False,
         "ddp_bucket_cap_mb": 128},
        id="compiled-autograd-overlap",
    ),
]


@TWO_GPUS
@pytest.mark.parametrize(("flags", "selected"), CASES)
def test_optimized_two_gpu_async_resume_preserves_full_state(tmp_path, flags, selected):
    (continuous, resumed), common = fixture_runs(tmp_path, ["continuous", "resumed"])
    common += [
        "--loss-mode", "compiled-full", "--compile-core", "--compile-mode", "default",
        "--bf16-residuals", "--no-activation-checkpointing", "--autocast-cache",
        "--eval-every", "2", "--save-every", "2", *flags,
    ]
    selected = dict(selected)
    if "__fixture_plan__" in common:
        from ops.matmul_execution import load_matmul_plan
        tile = {"block_m": 16, "block_n": 32, "block_k": 16, "num_warps": 4, "num_stages": 2}
        document = {"format": "warpstate-matmul-plan-v1", "entries": [
            {"kind": "swiglu", "m": 16, "n": 64, "k": 16,
             "a_stride": [16, 1], "b_stride": [1, 16], "dtype": "bfloat16",
             "algorithm": "persistent", "tile": tile},
            {"kind": "mm", "m": 64, "n": 16, "k": 16,
             "a_stride": [1, 64], "b_stride": [16, 1], "dtype": "bfloat16",
             "algorithm": "persistent_tma", "tile": tile},
        ]}
        recipe = tmp_path / "matmul-plan.json"
        recipe.write_text(json.dumps(document))
        common[common.index("__fixture_plan__")] = str(recipe)
        selected["matmul_execution"] = load_matmul_plan(recipe).metadata
    initial = torch.load(tmp_path / "initial.pt", map_location="cpu", weights_only=False)
    # The 73-token fixture consumes 32, 32, then 9 tokens. On its final update,
    # rank zero consumes the remaining tokens and rank one has entirely masked
    # targets. Resume must preserve that weighting and every rank's RNG state.
    expected = run_training(continuous, common, 2, timeout=300,
                            expect_validation_display=True)
    async_common = common + ["--async-checkpoint"]
    partial = run_training(resumed, async_common, 2, ["--max-steps", "1"],
                          timeout=300, expect_async=True, expect_validation_display=True)
    assert partial["tokens_seen"] == 32 and partial["step"] == 1
    assert partial["val_step"] == 1 and partial["val_tokens"] == 32
    actual = run_training(resumed, async_common, 2, timeout=300, expect_async=True,
                         expect_validation_display=True)

    assert actual["format"] == expected["format"] == "warpstate-server-v1"
    assert actual["step"] == expected["step"] == 3
    assert actual["tokens_seen"] == expected["tokens_seen"] == 73
    assert actual["origin"] == partial["origin"] == expected["origin"]
    assert actual["execution_history"] == partial["execution_history"] == expected["execution_history"]
    assert actual["signature"] == partial["signature"] == expected["signature"]
    settings = actual["signature"]["settings"]
    for name, value in selected.items():
        assert settings[name] == value
    assert settings["world_size"] == 2
    assert settings["compile_core"] is True
    assert settings["compile_mode"] == "default"
    assert settings["loss_mode"] == "compiled-full"
    assert settings["bf16_residuals"] is True
    assert settings["activation_checkpointing"] is False
    assert settings["seq_len"] * settings["micro_batch"] * settings["grad_accum"] * settings["world_size"] == 32
    assert len(actual["rng_by_rank"]) == len(partial["rng_by_rank"]) == 2
    for name in ("model", "optimizer", "rng_by_rank"):
        assert_tensors_equal(actual[name], expected[name])

    assert actual["model"].keys() == initial["model"].keys()
    assert all(not name.startswith(("module.", "_orig_mod.")) for name in actual["model"])
    assert all(value.dtype == torch.float32 and torch.isfinite(value).all()
               for value in actual["model"].values())
    assert actual["optimizer"]["state"]
    for state in actual["optimizer"]["state"].values():
        assert state["step"].item() == 3
        for name in ("exp_avg", "exp_avg_sq"):
            assert state[name].dtype == torch.float32 and torch.isfinite(state[name]).all()
    assert actual["val_loss"] == expected["val_loss"]
    assert math.isfinite(actual["val_loss"])
    assert actual["val_step"] == expected["val_step"] == 3
    assert actual["val_tokens"] == expected["val_tokens"] == 73

    for directory, async_enabled in [(continuous, False), (resumed, True)]:
        runtime = json.loads((directory / "run/settings.json").read_text())["runtime"]
        for name, value in selected.items():
            assert runtime[name] == value
        assert runtime["async_checkpoint"] is async_enabled
        assert runtime["bf16_residuals"] is True
        assert runtime["eval_every"] == runtime["save_every"] == 2
        assert not (directory / "run/latest.pt.tmp").exists()
        records = [json.loads(line) for line in
                   (directory / "run/metrics.jsonl").read_text().splitlines()]
        assert [record["step"] for record in records] == [1, 2, 3]
        assert [record["tokens"] for record in records] == [32, 64, 73]
        assert [record["lr"] for record in records] == [0.001, 0.002, 0.002]
        assert all(math.isfinite(record["loss"]) and math.isfinite(record["grad_norm"])
                   for record in records)
    validations = [json.loads(line) for line in
                   (resumed / "run/validation.jsonl").read_text().splitlines()]
    assert [record["tokens"] for record in validations] == [32, 64, 73]
    assert all(math.isfinite(record["loss"]) for record in validations)