File size: 6,549 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
"""Backward layout selection must be explicit, applicable, and resume-visible."""

from copy import deepcopy
from pathlib import Path
import sys

import pytest

import run_40b
from ops import benchmark_training


@pytest.fixture(params=[run_40b, benchmark_training], ids=["runner", "benchmark"])
def cli(request, monkeypatch):
    workspace = Path(__file__).resolve().parents[1]
    for name in ("TMPDIR", "XDG_CACHE_HOME", "CUDA_CACHE_PATH", "TRITON_CACHE_DIR",
                 "TORCHINDUCTOR_CACHE_DIR"):
        monkeypatch.setenv(name, str(workspace / "tmp"))
    monkeypatch.delenv("WORLD_SIZE", raising=False)
    compile_flag = "--compile-core" if request.param is run_40b else "--compile-cores"

    def parse(flags=(), *, supported=False, omit=()):
        common = ["--micro-batch", "4"]
        if supported:
            common += ["--gradient-mode", "compiled-autograd-overlap", compile_flag,
                       "--no-activation-checkpointing", "--bf16-residuals"]
        common = [flag for flag in common if flag not in omit]
        monkeypatch.setattr(sys, "argv", [request.param.__file__, *common, *flags])
        return request.param.arguments()

    return parse, compile_flag


def test_baseline_keeps_existing_cli_usable(cli):
    parse, _ = cli
    args = parse()
    assert args.backward_layout == "baseline"
    assert args.gradient_mode == "eager"


@pytest.mark.parametrize("mode", ["inspect", "paired"])
def test_supported_selection_requires_an_explicit_flag(cli, mode):
    parse, _ = cli
    args = parse(["--backward-layout", mode], supported=True)
    assert args.backward_layout == mode
    assert args.gradient_mode == "compiled-autograd-overlap"
    assert args.bf16_residuals
    assert not args.activation_checkpointing


@pytest.mark.parametrize("flags", [
    ["--gradient-mode", "eager"],
    ["--gradient-mode", "compiled-autograd"],
    ["--activation-checkpointing"],
    ["--micro-batch", "2"],
    ["--seq-len", "1024"],
    ["--activation-mode", "swiglu"],
    ["--matmul-plan", "/workspace/WarpState/rtx6000_40b/configs/not-loaded.json"],
    ["--cache-core-weight-casts"],
    ["--compile-mode", "reduce-overhead"],
    ["--compile-mode", "max-autotune-no-cudagraphs"],
    ["--ddp-batched-grad-copy"],
])
@pytest.mark.parametrize("mode", ["inspect", "paired"])
def test_reject_unsupported_selection_before_loading_cuda(cli, flags, mode):
    parse, _ = cli
    with pytest.raises(SystemExit) as error:
        parse(["--backward-layout", mode, *flags], supported=True)
    assert error.value.code == 2


@pytest.mark.parametrize("missing", ["compile", "bf16"])
@pytest.mark.parametrize("mode", ["inspect", "paired"])
def test_reject_missing_required_execution_setting(cli, missing, mode):
    parse, compile_flag = cli
    omit = (compile_flag,) if missing == "compile" else ("--bf16-residuals",)
    with pytest.raises(SystemExit) as error:
        parse(["--backward-layout", mode], supported=True, omit=omit)
    assert error.value.code == 2


@pytest.mark.parametrize("mode", ["inspect", "paired"])
def test_benchmark_rejects_unvalidated_fused_normalization(monkeypatch, capsys, mode):
    monkeypatch.setattr(sys, "argv", [benchmark_training.__file__, "--micro-batch", "4",
                       "--gradient-mode", "compiled-autograd-overlap", "--compile-cores",
                       "--bf16-residuals", "--backward-layout", mode, "--fused-rms-norm"])
    with pytest.raises(SystemExit) as error:
        benchmark_training.arguments()
    assert error.value.code == 2
    assert "--backward-layout requires" in capsys.readouterr().err


def runner_signature(monkeypatch, metadata):
    monkeypatch.delenv("WORLD_SIZE", raising=False)
    monkeypatch.setattr(sys, "argv", [run_40b.__file__, "--micro-batch", "4"])
    args = run_40b.arguments()
    args.backward_layout_metadata = metadata
    return run_40b.train_signature(args, {"recipe": "fixed-corpus"})


def test_signature_records_layout_implementation_identity(monkeypatch):
    metadata = {"mode": "paired", "identity": "implementation-a",
                "source_sha256": {"ops/backward_layout.py": "kernel-a"},
                "block": 256, "warps": 8}
    expected = deepcopy(metadata)
    signature = runner_signature(monkeypatch, metadata)
    assert signature["settings"]["backward_layout_execution"] == expected
    assert metadata == expected


def test_legacy_checkpoint_remains_compatible_with_baseline(monkeypatch):
    current = runner_signature(monkeypatch, None)
    legacy = deepcopy(current)
    del legacy["settings"]["backward_layout_execution"]
    before = deepcopy(legacy)
    assert run_40b.check_resume_signature(legacy, current) is False
    assert legacy == before


@pytest.mark.parametrize("change", [
    {"mode": "inspect", "identity": "implementation-a"},
    {"mode": "paired", "identity": "implementation-a"},
    {"mode": "paired", "identity": "implementation-b"},
    {"mode": "paired", "identity": "implementation-a", "source_sha256": {"kernel": "changed"}},
    None,
])
def test_execution_identity_changes_require_reconfiguration(monkeypatch, change):
    original = runner_signature(monkeypatch, {"mode": "paired", "identity": "implementation-a",
                                             "source_sha256": {"kernel": "original"}})
    candidate = runner_signature(monkeypatch, change)
    before = deepcopy((original, candidate))
    with pytest.raises(ValueError, match="Resume settings/data differ"):
        run_40b.check_resume_signature(original, candidate)
    assert run_40b.check_resume_signature(original, candidate, reconfigure=True) is True
    assert (original, candidate) == before


def test_same_layout_identity_does_not_require_reconfiguration(monkeypatch):
    original = runner_signature(monkeypatch, {"mode": "paired", "identity": "implementation-a"})
    assert run_40b.check_resume_signature(original, deepcopy(original)) is False


@pytest.mark.parametrize("change", ["data", "tokens", "batch"])
def test_layout_reconfiguration_cannot_change_training_objective(monkeypatch, change):
    original = runner_signature(monkeypatch, None)
    candidate = runner_signature(monkeypatch, {"mode": "paired", "identity": "implementation-a"})
    if change == "data":
        candidate["data"]["recipe"] = "another-corpus"
    elif change == "tokens":
        candidate["settings"]["train_tokens"] += 1
    else:
        candidate["settings"]["grad_accum"] += 1
    with pytest.raises(ValueError):
        run_40b.check_resume_signature(original, candidate, reconfigure=True)