File size: 3,325 Bytes
c8ba9c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
"""Both entry points validate and record communication experiment controls."""

import sys
from pathlib import Path

import pytest

import run_40b
from ops import benchmark_training


@pytest.fixture(params=[run_40b, benchmark_training], ids=["runner", "benchmark"])
def cli(request, monkeypatch):
    workspace = Path(__file__).resolve().parents[1]
    for name in ("TMPDIR", "XDG_CACHE_HOME", "CUDA_CACHE_PATH", "TRITON_CACHE_DIR",
                 "TORCHINDUCTOR_CACHE_DIR"):
        monkeypatch.setenv(name, str(workspace / "tmp"))
    monkeypatch.delenv("WORLD_SIZE", raising=False)

    def parse(flags):
        monkeypatch.setattr(sys, "argv", [request.param.__file__, "--micro-batch", "4", *flags])
        return request.param.arguments()

    return parse


def test_current_overlap_behavior_remains_the_default(cli):
    args = cli(["--gradient-mode", "compiled-autograd-overlap"])
    assert not args.overlap_swap_collectives
    assert not args.overlap_runtime_estimates
    assert not args.overlap_producer_schedule
    assert args.overlap_memory_budget == 0


def test_overlap_experiment_is_explicit_and_in_resume_signature(cli):
    args = cli(["--gradient-mode", "compiled-autograd-overlap",
                "--overlap-swap-collectives", "--overlap-runtime-estimates",
                "--overlap-memory-budget", "0.02"])
    assert args.overlap_swap_collectives and args.overlap_runtime_estimates
    assert args.overlap_memory_budget == 0.02
    if hasattr(args, "grad_accum"):
        settings = run_40b.train_signature(args, {})["settings"]
        assert settings["overlap_swap_collectives"] is True
        assert settings["overlap_runtime_estimates"] is True
        assert settings["overlap_memory_budget"] == 0.02


def test_producer_scheduling_is_explicit_and_recorded(cli):
    args = cli(["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule"])
    assert args.overlap_producer_schedule is True
    assert not args.overlap_swap_collectives and not args.overlap_runtime_estimates
    assert args.overlap_memory_budget == 0.0
    if hasattr(args, "grad_accum"):
        assert run_40b.train_signature(args, {})["settings"]["overlap_producer_schedule"] is True


@pytest.mark.parametrize("flags", [
    ["--overlap-swap-collectives"],
    ["--gradient-mode", "compiled-autograd", "--overlap-runtime-estimates"],
    ["--overlap-memory-budget", "0.02"],
    ["--gradient-mode", "compiled-autograd-overlap", "--overlap-memory-budget", "nan"],
    ["--gradient-mode", "compiled-autograd-overlap", "--overlap-memory-budget", "-0.02"],
    ["--gradient-mode", "compiled-autograd-overlap", "--overlap-memory-budget", "0.26"],
    ["--overlap-producer-schedule"],
    ["--gradient-mode", "compiled-autograd", "--overlap-producer-schedule"],
    ["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule",
     "--overlap-swap-collectives"],
    ["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule",
     "--overlap-runtime-estimates"],
    ["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule",
     "--overlap-memory-budget", "0.02"],
])
def test_reject_silent_noops_and_unbounded_memory_before_cuda(cli, flags):
    with pytest.raises(SystemExit) as error:
        cli(flags)
    assert error.value.code == 2