"""Absolute training stops preserve the objective, checkpoint, and continuation.""" import json import math import sys import numpy as np import pytest import torch from test_training_execution import ( TWO_GPUS, assert_tensors_equal, fixture_runs, pipeline, run_training, ) from warpstate import WarpStateConfig, WarpStateLM def test_stop_at_step_cli_defaults_to_no_absolute_limit(monkeypatch): monkeypatch.setattr(sys, "argv", ["run_40b.py"]) args = pipeline.arguments() assert args.stop_at_step is None assert args.max_steps is None assert args.train_tokens == 40_000_000_000 @pytest.mark.parametrize("target", [1, 64_000, 2_048_000]) def test_stop_at_step_cli_accepts_positive_absolute_target(monkeypatch, target): monkeypatch.setattr(sys, "argv", ["run_40b.py", "--stop-at-step", str(target)]) args = pipeline.arguments() assert args.stop_at_step == target assert args.max_steps is None @pytest.mark.parametrize("target", ["0", "-1", "2.5", "nan"]) def test_stop_at_step_cli_rejects_nonpositive_or_noninteger_target(monkeypatch, target): monkeypatch.setattr(sys, "argv", ["run_40b.py", "--stop-at-step", target]) with pytest.raises(SystemExit) as error: pipeline.arguments() assert error.value.code == 2 def test_stop_at_step_and_additional_limit_preserve_resume_signature(monkeypatch): manifest = {"complete": True, "signature": {"source": "unchanged-fixture"}} monkeypatch.setattr(sys, "argv", ["run_40b.py"]) expected = pipeline.train_signature(pipeline.arguments(), manifest) for flags in (["--stop-at-step", "64000"], ["--stop-at-step", "64000", "--max-steps", "1"], ["--stop-at-step", "96000"], []): monkeypatch.setattr(sys, "argv", ["run_40b.py", *flags]) args = pipeline.arguments() actual = pipeline.train_signature(args, manifest) assert actual == expected assert actual["settings"]["train_tokens"] == 40_000_000_000 assert "stop_at_step" not in actual["settings"] assert "max_steps" not in actual["settings"] assert pipeline.check_resume_signature(expected, actual) is False if "--max-steps" in flags: assert args.max_steps == 1 and args.stop_at_step == 64000 def _records(directory, name): return [json.loads(line) for line in (directory / "run" / name).read_text().splitlines()] @pytest.mark.parametrize(("world_size", "async_checkpoint"), [ pytest.param(1, False, id="single-gpu-sync", marks=pytest.mark.skipif(not torch.cuda.is_available(), reason="CUDA required")), pytest.param(2, True, id="two-gpu-async", marks=TWO_GPUS), ]) def test_absolute_stop_checkpoint_and_unlimited_resume_preserve_full_state( tmp_path, monkeypatch, world_size, async_checkpoint): (continuous, resumed), common = fixture_runs(tmp_path, ["continuous", "resumed"]) # The original two-head tiny fixture showed a one-ULP step-two grad-norm # difference after eager DDP resume, reproduced exactly with --max-steps # alone. Its two-element parameter offsets may change foreach-norm alignment # when DDP rebuilds buckets. Four heads make every parameter size divisible # by four and retain strict uninterrupted-reference equality below, without # changing training code or weakening tolerances. config_path = tmp_path / "config.json" config = WarpStateConfig(**json.loads(config_path.read_text())) config.n_heads = 4 config.validate() config.to_json(config_path) torch.manual_seed(133) initial = WarpStateLM(config) assert all(parameter.numel() % 4 == 0 for parameter in initial.parameters()) torch.save({"config": vars(config), "model": initial.state_dict(), "tokens_seen": 100}, tmp_path / "initial.pt") # Four optimizer updates consume 32, 32, 32, then 9 tokens. Target three # falls outside the ten-step evaluation/save cadence, after cosine decay # begins. Replacing the full LR horizon with the pause target must fail. # Two GPUs also exercise an entirely masked final rank on the last update. common += ["--loss-mode", "full", "--bf16-residuals", "--no-activation-checkpointing", "--eval-every", "10", "--save-every", "10", "--train-tokens", "105", "--warmup-steps", "1"] with monkeypatch.context() as context: context.setattr(sys, "argv", common) data_signature = pipeline.data_signature(pipeline.arguments()) for directory in (continuous, resumed): np.arange(105, dtype="