File size: 8,466 Bytes
c8ba9c6 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 | """Two-process CPU/Gloo regressions for coordinated checkpoint publication."""
from datetime import timedelta
import json
import os
from pathlib import Path
import subprocess
import sys
import threading
import pytest
import torch
import torch.distributed as dist
from ops import checkpoint_writer
def _worker(rank, directory, scenario):
directory = Path(directory)
report = {"rank": rank, "writes": [], "syncs": 0, "factory_calls": 0}
writer = None
release = threading.Event()
original_write = checkpoint_writer.atomic_torch_save
original_snapshot = checkpoint_writer.snapshot_to_cpu
original_fsync = os.fsync
original_broadcast = dist.broadcast
def save(path, state):
report["writes"].append({
"rank": rank, "thread": threading.current_thread().name,
"device": str(state["weight"].device),
})
if scenario in {"check", "submit", "flush", "close"}:
raise OSError("injected checkpoint disk failure")
assert release.wait(10), "Test did not release checkpoint serialization"
original_write(path, state)
def snapshot(state):
if scenario == "snapshot":
raise MemoryError("injected CPU snapshot failure")
return original_snapshot(state)
def fsync(descriptor):
original_fsync(descriptor)
report["syncs"] += 1
def forbidden_collective(*args, **kwargs):
raise AssertionError("Exceptional context exit attempted a collective")
checkpoint_writer.atomic_torch_save = save
checkpoint_writer.snapshot_to_cpu = snapshot
os.fsync = fsync
try:
dist.init_process_group(
"gloo", init_method=(directory / "rendezvous").as_uri(),
rank=rank, world_size=2, timeout=timedelta(seconds=15),
)
try:
with checkpoint_writer.DistributedCheckpointWriter(
directory / "latest.pt", rank=rank,
device=torch.device("cpu"), distributed=True,
) as writer:
report["owns_writer"] = writer._writer is not None
state = {"weight": torch.tensor([1.0, 2.0]), "metadata": [3]}
def build_state():
report["factory_calls"] += 1
assert rank == 0, "Only rank zero may construct checkpoint state"
if scenario == "factory":
raise ValueError("injected checkpoint construction failure")
return state
report["staging_seconds"] = (
writer.submit(state_factory=build_state)
if scenario in {"factory", "normal_factory"}
else writer.submit(state if rank == 0 else None)
)
if scenario in {"check", "submit", "flush", "close"}:
# Ensure the disk error has happened before asking every rank
# to observe it through each distinct public operation.
if rank == 0:
try:
writer._writer._future.result(timeout=10)
except OSError:
pass
dist.barrier()
if scenario == "submit":
writer.submit(state if rank == 0 else None)
else:
getattr(writer, scenario)()
raise AssertionError("Checkpoint failure was not raised")
state["weight"].add_(10)
state["metadata"].append(4)
writer.check() # A pending write is allowed to overlap training.
report["exists_before_release"] = (directory / "latest.pt").exists()
dist.barrier()
release.set()
if scenario == "exceptional_exit":
dist.broadcast = forbidden_collective
raise ValueError("injected training failure")
report["normal_exit"] = True
except BaseException as error:
report["error"] = {"type": type(error).__name__, "message": str(error)}
if writer is not None:
report["closed"] = writer._writer is None or writer._writer._closed
finally:
release.set()
checkpoint_writer.atomic_torch_save = original_write
checkpoint_writer.snapshot_to_cpu = original_snapshot
os.fsync = original_fsync
dist.broadcast = original_broadcast
if dist.is_initialized():
dist.destroy_process_group()
(directory / f"rank-{rank}.json").write_text(json.dumps(report))
def _run_two_ranks(directory, scenario):
environment = os.environ.copy()
environment.update(CUDA_VISIBLE_DEVICES="", OMP_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1")
processes = []
try:
for rank in range(2):
processes.append(subprocess.Popen(
[sys.executable, str(Path(__file__).resolve()),
str(rank), str(directory), scenario],
env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True,
))
for rank, process in enumerate(processes):
output, _ = process.communicate(timeout=40)
assert process.returncode == 0, f"Rank {rank} failed:\n{output}"
finally:
for process in processes:
if process.poll() is None:
process.kill()
for process in processes:
process.wait(timeout=10)
return [json.loads((directory / f"rank-{rank}.json").read_text()) for rank in range(2)]
@pytest.mark.parametrize("operation", ["check", "submit", "flush", "close", "snapshot", "factory"])
def test_rank_zero_failure_reaches_both_ranks_without_hanging(tmp_path, operation):
path = tmp_path / "latest.pt"
torch.save({"old_checkpoint": True}, path)
reports = _run_two_ranks(tmp_path, operation)
expected_type, expected_message = {
"snapshot": ("MemoryError", "CPU snapshot failure"),
"factory": ("ValueError", "checkpoint construction failure"),
}.get(operation, ("OSError", "checkpoint disk failure"))
assert reports[0]["error"]["type"] == expected_type
assert reports[1]["error"]["type"] == "RuntimeError"
for rank, report in enumerate(reports):
assert expected_message in report["error"]["message"]
assert report["owns_writer"] is (rank == 0)
assert report["closed"]
assert reports[1]["writes"] == []
assert reports[1]["factory_calls"] == 0
assert reports[0]["factory_calls"] == int(operation == "factory")
assert torch.load(path, weights_only=False) == {"old_checkpoint": True}
assert not path.with_suffix(".pt.tmp").exists()
@pytest.mark.parametrize("scenario", ["normal_exit", "normal_factory"])
def test_normal_exit_publishes_one_immutable_cpu_snapshot_durably(tmp_path, scenario):
reports = _run_two_ranks(tmp_path, scenario)
for rank, report in enumerate(reports):
assert "error" not in report, report
assert report["owns_writer"] is (rank == 0)
assert report["normal_exit"] and report["closed"]
assert report["exists_before_release"] is False
assert reports[0]["staging_seconds"] >= 0
assert reports[1]["staging_seconds"] is None
assert reports[1]["factory_calls"] == 0
assert reports[0]["factory_calls"] == int(scenario == "normal_factory")
assert reports[1]["writes"] == []
assert reports[1]["syncs"] == 0
assert reports[0]["syncs"] == 2 # File contents, then directory publication.
assert len(reports[0]["writes"]) == 1
write = reports[0]["writes"][0]
assert write["device"] == "cpu" and write["thread"].startswith("checkpoint-writer")
state = torch.load(tmp_path / "latest.pt", weights_only=False)
torch.testing.assert_close(state["weight"], torch.tensor([1.0, 2.0]), rtol=0, atol=0)
assert state["metadata"] == [3]
assert not (tmp_path / "latest.pt.tmp").exists()
def test_exceptional_exit_joins_local_writer_without_any_collectives(tmp_path):
reports = _run_two_ranks(tmp_path, "exceptional_exit")
for report in reports:
assert report["error"] == {"type": "ValueError", "message": "injected training failure"}
assert report["closed"]
assert (tmp_path / "latest.pt").is_file()
assert reports[1]["writes"] == []
if __name__ == "__main__":
_worker(int(sys.argv[1]), sys.argv[2], sys.argv[3])
|