Download test_distributed_checkpoint_writer.py from zerdovzad/WarpStateAI: direct link, hf CLI and curl.
- Browser
- Download file 8.47 kB
-
https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_distributed_checkpoint_writer.py
- Command line
-
hf download hf://zerdovzad/WarpStateAI/test_distributed_checkpoint_writer.py
-
curl -L -o test_distributed_checkpoint_writer.py https://huggingface.co/zerdovzad/WarpStateAI/resolve/main/test_distributed_checkpoint_writer.py
8.47 kB
| """Two-process CPU/Gloo regressions for coordinated checkpoint publication.""" | |
| from datetime import timedelta | |
| import json | |
| import os | |
| from pathlib import Path | |
| import subprocess | |
| import sys | |
| import threading | |
| import pytest | |
| import torch | |
| import torch.distributed as dist | |
| from ops import checkpoint_writer | |
| def _worker(rank, directory, scenario): | |
| directory = Path(directory) | |
| report = {"rank": rank, "writes": [], "syncs": 0, "factory_calls": 0} | |
| writer = None | |
| release = threading.Event() | |
| original_write = checkpoint_writer.atomic_torch_save | |
| original_snapshot = checkpoint_writer.snapshot_to_cpu | |
| original_fsync = os.fsync | |
| original_broadcast = dist.broadcast | |
| def save(path, state): | |
| report["writes"].append({ | |
| "rank": rank, "thread": threading.current_thread().name, | |
| "device": str(state["weight"].device), | |
| }) | |
| if scenario in {"check", "submit", "flush", "close"}: | |
| raise OSError("injected checkpoint disk failure") | |
| assert release.wait(10), "Test did not release checkpoint serialization" | |
| original_write(path, state) | |
| def snapshot(state): | |
| if scenario == "snapshot": | |
| raise MemoryError("injected CPU snapshot failure") | |
| return original_snapshot(state) | |
| def fsync(descriptor): | |
| original_fsync(descriptor) | |
| report["syncs"] += 1 | |
| def forbidden_collective(*args, **kwargs): | |
| raise AssertionError("Exceptional context exit attempted a collective") | |
| checkpoint_writer.atomic_torch_save = save | |
| checkpoint_writer.snapshot_to_cpu = snapshot | |
| os.fsync = fsync | |
| try: | |
| dist.init_process_group( | |
| "gloo", init_method=(directory / "rendezvous").as_uri(), | |
| rank=rank, world_size=2, timeout=timedelta(seconds=15), | |
| ) | |
| try: | |
| with checkpoint_writer.DistributedCheckpointWriter( | |
| directory / "latest.pt", rank=rank, | |
| device=torch.device("cpu"), distributed=True, | |
| ) as writer: | |
| report["owns_writer"] = writer._writer is not None | |
| state = {"weight": torch.tensor([1.0, 2.0]), "metadata": [3]} | |
| def build_state(): | |
| report["factory_calls"] += 1 | |
| assert rank == 0, "Only rank zero may construct checkpoint state" | |
| if scenario == "factory": | |
| raise ValueError("injected checkpoint construction failure") | |
| return state | |
| report["staging_seconds"] = ( | |
| writer.submit(state_factory=build_state) | |
| if scenario in {"factory", "normal_factory"} | |
| else writer.submit(state if rank == 0 else None) | |
| ) | |
| if scenario in {"check", "submit", "flush", "close"}: | |
| # Ensure the disk error has happened before asking every rank | |
| # to observe it through each distinct public operation. | |
| if rank == 0: | |
| try: | |
| writer._writer._future.result(timeout=10) | |
| except OSError: | |
| pass | |
| dist.barrier() | |
| if scenario == "submit": | |
| writer.submit(state if rank == 0 else None) | |
| else: | |
| getattr(writer, scenario)() | |
| raise AssertionError("Checkpoint failure was not raised") | |
| state["weight"].add_(10) | |
| state["metadata"].append(4) | |
| writer.check() # A pending write is allowed to overlap training. | |
| report["exists_before_release"] = (directory / "latest.pt").exists() | |
| dist.barrier() | |
| release.set() | |
| if scenario == "exceptional_exit": | |
| dist.broadcast = forbidden_collective | |
| raise ValueError("injected training failure") | |
| report["normal_exit"] = True | |
| except BaseException as error: | |
| report["error"] = {"type": type(error).__name__, "message": str(error)} | |
| if writer is not None: | |
| report["closed"] = writer._writer is None or writer._writer._closed | |
| finally: | |
| release.set() | |
| checkpoint_writer.atomic_torch_save = original_write | |
| checkpoint_writer.snapshot_to_cpu = original_snapshot | |
| os.fsync = original_fsync | |
| dist.broadcast = original_broadcast | |
| if dist.is_initialized(): | |
| dist.destroy_process_group() | |
| (directory / f"rank-{rank}.json").write_text(json.dumps(report)) | |
| def _run_two_ranks(directory, scenario): | |
| environment = os.environ.copy() | |
| environment.update(CUDA_VISIBLE_DEVICES="", OMP_NUM_THREADS="1", OPENBLAS_NUM_THREADS="1") | |
| processes = [] | |
| try: | |
| for rank in range(2): | |
| processes.append(subprocess.Popen( | |
| [sys.executable, str(Path(__file__).resolve()), | |
| str(rank), str(directory), scenario], | |
| env=environment, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, | |
| )) | |
| for rank, process in enumerate(processes): | |
| output, _ = process.communicate(timeout=40) | |
| assert process.returncode == 0, f"Rank {rank} failed:\n{output}" | |
| finally: | |
| for process in processes: | |
| if process.poll() is None: | |
| process.kill() | |
| for process in processes: | |
| process.wait(timeout=10) | |
| return [json.loads((directory / f"rank-{rank}.json").read_text()) for rank in range(2)] | |
| def test_rank_zero_failure_reaches_both_ranks_without_hanging(tmp_path, operation): | |
| path = tmp_path / "latest.pt" | |
| torch.save({"old_checkpoint": True}, path) | |
| reports = _run_two_ranks(tmp_path, operation) | |
| expected_type, expected_message = { | |
| "snapshot": ("MemoryError", "CPU snapshot failure"), | |
| "factory": ("ValueError", "checkpoint construction failure"), | |
| }.get(operation, ("OSError", "checkpoint disk failure")) | |
| assert reports[0]["error"]["type"] == expected_type | |
| assert reports[1]["error"]["type"] == "RuntimeError" | |
| for rank, report in enumerate(reports): | |
| assert expected_message in report["error"]["message"] | |
| assert report["owns_writer"] is (rank == 0) | |
| assert report["closed"] | |
| assert reports[1]["writes"] == [] | |
| assert reports[1]["factory_calls"] == 0 | |
| assert reports[0]["factory_calls"] == int(operation == "factory") | |
| assert torch.load(path, weights_only=False) == {"old_checkpoint": True} | |
| assert not path.with_suffix(".pt.tmp").exists() | |
| def test_normal_exit_publishes_one_immutable_cpu_snapshot_durably(tmp_path, scenario): | |
| reports = _run_two_ranks(tmp_path, scenario) | |
| for rank, report in enumerate(reports): | |
| assert "error" not in report, report | |
| assert report["owns_writer"] is (rank == 0) | |
| assert report["normal_exit"] and report["closed"] | |
| assert report["exists_before_release"] is False | |
| assert reports[0]["staging_seconds"] >= 0 | |
| assert reports[1]["staging_seconds"] is None | |
| assert reports[1]["factory_calls"] == 0 | |
| assert reports[0]["factory_calls"] == int(scenario == "normal_factory") | |
| assert reports[1]["writes"] == [] | |
| assert reports[1]["syncs"] == 0 | |
| assert reports[0]["syncs"] == 2 # File contents, then directory publication. | |
| assert len(reports[0]["writes"]) == 1 | |
| write = reports[0]["writes"][0] | |
| assert write["device"] == "cpu" and write["thread"].startswith("checkpoint-writer") | |
| state = torch.load(tmp_path / "latest.pt", weights_only=False) | |
| torch.testing.assert_close(state["weight"], torch.tensor([1.0, 2.0]), rtol=0, atol=0) | |
| assert state["metadata"] == [3] | |
| assert not (tmp_path / "latest.pt.tmp").exists() | |
| def test_exceptional_exit_joins_local_writer_without_any_collectives(tmp_path): | |
| reports = _run_two_ranks(tmp_path, "exceptional_exit") | |
| for report in reports: | |
| assert report["error"] == {"type": "ValueError", "message": "injected training failure"} | |
| assert report["closed"] | |
| assert (tmp_path / "latest.pt").is_file() | |
| assert reports[1]["writes"] == [] | |
| if __name__ == "__main__": | |
| _worker(int(sys.argv[1]), sys.argv[2], sys.argv[3]) | |