File size: 3,325 Bytes
c8ba9c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 | """Both entry points validate and record communication experiment controls."""
import sys
from pathlib import Path
import pytest
import run_40b
from ops import benchmark_training
@pytest.fixture(params=[run_40b, benchmark_training], ids=["runner", "benchmark"])
def cli(request, monkeypatch):
workspace = Path(__file__).resolve().parents[1]
for name in ("TMPDIR", "XDG_CACHE_HOME", "CUDA_CACHE_PATH", "TRITON_CACHE_DIR",
"TORCHINDUCTOR_CACHE_DIR"):
monkeypatch.setenv(name, str(workspace / "tmp"))
monkeypatch.delenv("WORLD_SIZE", raising=False)
def parse(flags):
monkeypatch.setattr(sys, "argv", [request.param.__file__, "--micro-batch", "4", *flags])
return request.param.arguments()
return parse
def test_current_overlap_behavior_remains_the_default(cli):
args = cli(["--gradient-mode", "compiled-autograd-overlap"])
assert not args.overlap_swap_collectives
assert not args.overlap_runtime_estimates
assert not args.overlap_producer_schedule
assert args.overlap_memory_budget == 0
def test_overlap_experiment_is_explicit_and_in_resume_signature(cli):
args = cli(["--gradient-mode", "compiled-autograd-overlap",
"--overlap-swap-collectives", "--overlap-runtime-estimates",
"--overlap-memory-budget", "0.02"])
assert args.overlap_swap_collectives and args.overlap_runtime_estimates
assert args.overlap_memory_budget == 0.02
if hasattr(args, "grad_accum"):
settings = run_40b.train_signature(args, {})["settings"]
assert settings["overlap_swap_collectives"] is True
assert settings["overlap_runtime_estimates"] is True
assert settings["overlap_memory_budget"] == 0.02
def test_producer_scheduling_is_explicit_and_recorded(cli):
args = cli(["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule"])
assert args.overlap_producer_schedule is True
assert not args.overlap_swap_collectives and not args.overlap_runtime_estimates
assert args.overlap_memory_budget == 0.0
if hasattr(args, "grad_accum"):
assert run_40b.train_signature(args, {})["settings"]["overlap_producer_schedule"] is True
@pytest.mark.parametrize("flags", [
["--overlap-swap-collectives"],
["--gradient-mode", "compiled-autograd", "--overlap-runtime-estimates"],
["--overlap-memory-budget", "0.02"],
["--gradient-mode", "compiled-autograd-overlap", "--overlap-memory-budget", "nan"],
["--gradient-mode", "compiled-autograd-overlap", "--overlap-memory-budget", "-0.02"],
["--gradient-mode", "compiled-autograd-overlap", "--overlap-memory-budget", "0.26"],
["--overlap-producer-schedule"],
["--gradient-mode", "compiled-autograd", "--overlap-producer-schedule"],
["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule",
"--overlap-swap-collectives"],
["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule",
"--overlap-runtime-estimates"],
["--gradient-mode", "compiled-autograd-overlap", "--overlap-producer-schedule",
"--overlap-memory-budget", "0.02"],
])
def test_reject_silent_noops_and_unbounded_memory_before_cuda(cli, flags):
with pytest.raises(SystemExit) as error:
cli(flags)
assert error.value.code == 2
|