File size: 4,694 Bytes
aa02c28 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 | from __future__ import annotations
import multiprocessing as mp
from pathlib import Path
import torch
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel
from psi.trainers.posttrain_full_duplex import (
_all_reduce_sum_copy,
average_full_duplex_gradients,
build_dynamic_source_slot_plan,
active_source_schedule_offsets,
)
def test_metric_collective_does_not_mutate_rank_local_denominator(monkeypatch):
local_count = torch.tensor(7.0, requires_grad=True)
def fake_all_reduce(value, op):
assert op == dist.ReduceOp.SUM
value.mul_(8.0)
monkeypatch.setattr(dist, "all_reduce", fake_all_reduce)
global_count = _all_reduce_sum_copy(local_count, distributed=True)
assert global_count.item() == 56.0
assert local_count.item() == 7.0
assert global_count.data_ptr() != local_count.data_ptr()
assert global_count.requires_grad is False
metric_row = torch.stack(
[torch.tensor(5.0), local_count.detach()]
)
dist.all_reduce(metric_row, op=dist.ReduceOp.SUM)
assert torch.isclose(
metric_row[0] / metric_row[1],
torch.tensor(5.0 / 7.0),
)
def _ddp_worker(rank: int, init_file: str, result_queue) -> None:
dist.init_process_group(
"gloo",
init_method=f"file://{init_file}",
rank=rank,
world_size=2,
)
try:
# Rank 0's local timeline has a gap at 2,3; rank 1 still has real
# windows there. Both must keep the union and drop only the shared
# gap 4..19, including when their local timelines have different ends.
sources = [
{"source_id": str(i), "steps": [
{"update_semantic_cache": j == 0}
for j in range(2 if rank == 0 else 4)
]} for i in range(2)
]
plan = build_dynamic_source_slot_plan(sources, slot_count=1, semantic_cadence=20)
timeline = torch.tensor(plan.dense_steps)
dist.all_reduce(timeline, op=dist.ReduceOp.MAX)
offsets = active_source_schedule_offsets(
plan, int(timeline), device=torch.device("cpu"), distributed=True,
)
assert offsets == [0, 1, 2, 3, 20, 21, 22, 23]
torch.manual_seed(7)
model = DistributedDataParallel(
torch.nn.Linear(4, 2, bias=True),
broadcast_buffers=False,
find_unused_parameters=False,
)
optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
forwards = 1 if rank == 0 else 3
local_max = torch.tensor(forwards, dtype=torch.long)
dist.all_reduce(local_max, op=dist.ReduceOp.MAX)
loss = None
with model.no_sync():
for offset in range(int(local_max.item())):
if offset >= forwards:
# This rank is padded while the longer rank continues.
continue
inputs = torch.full((2, 4), float(rank + offset + 1))
value = model(inputs).square().mean() / float(forwards)
loss = value if loss is None else loss + value
assert loss is not None
# One backward after the complete local source, with no graph
# truncation or optimizer mutation inside it.
loss.backward()
average_full_duplex_gradients(
model.parameters(), world_size=2, bucket_cap_mb=1
)
optimizer.step()
optimizer.zero_grad()
flat = torch.cat([parameter.detach().reshape(-1) for parameter in model.parameters()])
gathered = [torch.empty_like(flat) for _ in range(2)]
dist.all_gather(gathered, flat)
if rank == 0:
result_queue.put(bool(torch.equal(gathered[0], gathered[1])))
finally:
dist.destroy_process_group()
def test_different_source_lengths_use_one_complete_source_collective(
tmp_path: Path,
):
init_file = tmp_path / "gloo_init"
context = mp.get_context("spawn")
result_queue = context.Queue()
processes = [
context.Process(
target=_ddp_worker,
args=(rank, str(init_file), result_queue),
)
for rank in range(2)
]
for process in processes:
process.start()
for process in processes:
# Spawned ranks import the full Transformers/Scipy stack from CPFS
# before entering Gloo. Cold FUSE directory scans alone can exceed
# 30 seconds even though the collective completes immediately.
process.join(timeout=180)
assert not process.is_alive(), "DDP gradient synchronization deadlocked"
assert process.exitcode == 0
assert result_queue.get(timeout=5) is True
|