File size: 8,884 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
"""Absolute training stops preserve the objective, checkpoint, and continuation."""

import json
import math
import sys

import numpy as np
import pytest
import torch

from test_training_execution import (
    TWO_GPUS,
    assert_tensors_equal,
    fixture_runs,
    pipeline,
    run_training,
)
from warpstate import WarpStateConfig, WarpStateLM


def test_stop_at_step_cli_defaults_to_no_absolute_limit(monkeypatch):
    monkeypatch.setattr(sys, "argv", ["run_40b.py"])
    args = pipeline.arguments()
    assert args.stop_at_step is None
    assert args.max_steps is None
    assert args.train_tokens == 40_000_000_000


@pytest.mark.parametrize("target", [1, 64_000, 2_048_000])
def test_stop_at_step_cli_accepts_positive_absolute_target(monkeypatch, target):
    monkeypatch.setattr(sys, "argv", ["run_40b.py", "--stop-at-step", str(target)])
    args = pipeline.arguments()
    assert args.stop_at_step == target
    assert args.max_steps is None


@pytest.mark.parametrize("target", ["0", "-1", "2.5", "nan"])
def test_stop_at_step_cli_rejects_nonpositive_or_noninteger_target(monkeypatch, target):
    monkeypatch.setattr(sys, "argv", ["run_40b.py", "--stop-at-step", target])
    with pytest.raises(SystemExit) as error:
        pipeline.arguments()
    assert error.value.code == 2


def test_stop_at_step_and_additional_limit_preserve_resume_signature(monkeypatch):
    manifest = {"complete": True, "signature": {"source": "unchanged-fixture"}}
    monkeypatch.setattr(sys, "argv", ["run_40b.py"])
    expected = pipeline.train_signature(pipeline.arguments(), manifest)
    for flags in (["--stop-at-step", "64000"],
                  ["--stop-at-step", "64000", "--max-steps", "1"],
                  ["--stop-at-step", "96000"], []):
        monkeypatch.setattr(sys, "argv", ["run_40b.py", *flags])
        args = pipeline.arguments()
        actual = pipeline.train_signature(args, manifest)
        assert actual == expected
        assert actual["settings"]["train_tokens"] == 40_000_000_000
        assert "stop_at_step" not in actual["settings"]
        assert "max_steps" not in actual["settings"]
        assert pipeline.check_resume_signature(expected, actual) is False
        if "--max-steps" in flags:
            assert args.max_steps == 1 and args.stop_at_step == 64000


def _records(directory, name):
    return [json.loads(line) for line in (directory / "run" / name).read_text().splitlines()]


@pytest.mark.parametrize(("world_size", "async_checkpoint"), [
    pytest.param(1, False, id="single-gpu-sync",
                 marks=pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")),
    pytest.param(2, True, id="two-gpu-async", marks=TWO_GPUS),
])
def test_absolute_stop_checkpoint_and_unlimited_resume_preserve_full_state(
        tmp_path, monkeypatch, world_size, async_checkpoint):
    (continuous, resumed), common = fixture_runs(tmp_path, ["continuous", "resumed"])
    # The original two-head tiny fixture showed a one-ULP step-two grad-norm
    # difference after eager DDP resume, reproduced exactly with --max-steps
    # alone. Its two-element parameter offsets may change foreach-norm alignment
    # when DDP rebuilds buckets. Four heads make every parameter size divisible
    # by four and retain strict uninterrupted-reference equality below, without
    # changing training code or weakening tolerances.
    config_path = tmp_path / "config.json"
    config = WarpStateConfig(**json.loads(config_path.read_text()))
    config.n_heads = 4
    config.validate()
    config.to_json(config_path)
    torch.manual_seed(133)
    initial = WarpStateLM(config)
    assert all(parameter.numel() % 4 == 0 for parameter in initial.parameters())
    torch.save({"config": vars(config), "model": initial.state_dict(), "tokens_seen": 100},
               tmp_path / "initial.pt")
    # Four optimizer updates consume 32, 32, 32, then 9 tokens. Target three
    # falls outside the ten-step evaluation/save cadence, after cosine decay
    # begins. Replacing the full LR horizon with the pause target must fail.
    # Two GPUs also exercise an entirely masked final rank on the last update.
    common += ["--loss-mode", "full", "--bf16-residuals", "--no-activation-checkpointing",
               "--eval-every", "10", "--save-every", "10",
               "--train-tokens", "105", "--warmup-steps", "1"]
    with monkeypatch.context() as context:
        context.setattr(sys, "argv", common)
        data_signature = pipeline.data_signature(pipeline.arguments())
    for directory in (continuous, resumed):
        np.arange(105, dtype="<u2").tofile(directory / "data/train.bin")
        manifest = directory / "data/manifest.json"
        document = json.loads(manifest.read_text())
        document["signature"] = data_signature
        manifest.write_text(json.dumps(document))
    expected = run_training(continuous, common, world_size, expect_validation_display=True)
    resumed_common = common + (["--async-checkpoint"] if async_checkpoint else [])
    stop_flags = ["--stop-at-step", "3"]

    partial = run_training(resumed, resumed_common, world_size,
                           [*stop_flags, "--max-steps", "1"], expect_async=async_checkpoint,
                           expect_validation_display=True)
    assert partial["step"] == partial["val_step"] == 1
    assert partial["tokens_seen"] == partial["val_tokens"] == 32
    runtime = json.loads((resumed / "run/settings.json").read_text())["runtime"]
    assert (runtime["stop_at_step"], runtime["max_steps"], runtime["end_step"]) == (3, 1, 1)

    # The same absolute target survives an earlier --max-steps pause. Treating
    # it as three additional updates would incorrectly finish all four here.
    stopped = run_training(resumed, resumed_common, world_size, stop_flags,
                           expect_async=async_checkpoint, expect_validation_display=True)
    assert stopped["format"] == "warpstate-server-v1"
    assert stopped["step"] == stopped["val_step"] == 3
    assert stopped["tokens_seen"] == stopped["val_tokens"] == 96
    assert stopped["optimizer"]["state"]
    assert all(state["step"].item() == 3 for state in stopped["optimizer"]["state"].values())
    assert len(stopped["rng_by_rank"]) == world_size
    assert stopped["signature"] == partial["signature"] == expected["signature"]
    assert stopped["execution_history"] == partial["execution_history"] == expected["execution_history"]
    assert "stop_at_step" not in stopped["signature"]["settings"]
    validations = _records(resumed, "validation.jsonl")
    assert [(r["step"], r["tokens"]) for r in validations] == [(1, 32), (3, 96)]
    assert stopped["val_loss"] == validations[-1]["loss"]
    assert math.isfinite(stopped["val_loss"])
    assert not (resumed / "run/latest.pt.tmp").exists()
    runtime = json.loads((resumed / "run/settings.json").read_text())["runtime"]
    assert (runtime["stop_at_step"], runtime["max_steps"], runtime["end_step"]) == (3, None, 3)

    # Starting at or already beyond the requested target must not execute a
    # speculative optimizer update. Compare all checkpoint fields, including
    # model, Adam, RNG, counters, validation metadata, and the objective.
    metrics_before = (resumed / "run/metrics.jsonl").read_bytes()
    for target in (3, 2):
        unchanged = run_training(resumed, resumed_common, world_size,
                                 ["--stop-at-step", str(target)])
        assert_tensors_equal(unchanged, stopped)
        assert (resumed / "run/metrics.jsonl").read_bytes() == metrics_before
        assert not (resumed / "run/latest.pt.tmp").exists()

    # Removing a stopping policy needs neither --reconfigure nor a changed LR
    # horizon; it must continue exactly as the uninterrupted reference did.
    actual = run_training(resumed, resumed_common, world_size,
                          expect_async=async_checkpoint, expect_validation_display=True)
    assert actual["step"] == actual["val_step"] == 4
    assert actual["tokens_seen"] == actual["val_tokens"] == 105
    assert_tensors_equal(actual, expected)
    runtime = json.loads((resumed / "run/settings.json").read_text())["runtime"]
    assert (runtime["stop_at_step"], runtime["max_steps"], runtime["end_step"]) == (None, None, 4)
    for directory in (continuous, resumed):
        records = _records(directory, "metrics.jsonl")
        assert [r["step"] for r in records] == [1, 2, 3, 4]
        assert [r["tokens"] for r in records] == [32, 64, 96, 105]
        assert [r["lr"] for r in records] == pytest.approx([0.002, 0.002, 0.00155, 0.00065],
                                                         rel=0, abs=1e-15)
        assert all(math.isfinite(r["loss"]) and math.isfinite(r["grad_norm"]) for r in records)
    assert [(r["step"], r["tokens"]) for r in _records(resumed, "validation.jsonl")] == [
        (1, 32), (3, 96), (4, 105)]