from __future__ import annotations import multiprocessing as mp from pathlib import Path import torch import torch.distributed as dist from torch.nn.parallel import DistributedDataParallel from psi.trainers.posttrain_full_duplex import ( _all_reduce_sum_copy, average_full_duplex_gradients, build_dynamic_source_slot_plan, active_source_schedule_offsets, ) def test_metric_collective_does_not_mutate_rank_local_denominator(monkeypatch): local_count = torch.tensor(7.0, requires_grad=True) def fake_all_reduce(value, op): assert op == dist.ReduceOp.SUM value.mul_(8.0) monkeypatch.setattr(dist, "all_reduce", fake_all_reduce) global_count = _all_reduce_sum_copy(local_count, distributed=True) assert global_count.item() == 56.0 assert local_count.item() == 7.0 assert global_count.data_ptr() != local_count.data_ptr() assert global_count.requires_grad is False metric_row = torch.stack( [torch.tensor(5.0), local_count.detach()] ) dist.all_reduce(metric_row, op=dist.ReduceOp.SUM) assert torch.isclose( metric_row[0] / metric_row[1], torch.tensor(5.0 / 7.0), ) def _ddp_worker(rank: int, init_file: str, result_queue) -> None: dist.init_process_group( "gloo", init_method=f"file://{init_file}", rank=rank, world_size=2, ) try: # Rank 0's local timeline has a gap at 2,3; rank 1 still has real # windows there. Both must keep the union and drop only the shared # gap 4..19, including when their local timelines have different ends. sources = [ {"source_id": str(i), "steps": [ {"update_semantic_cache": j == 0} for j in range(2 if rank == 0 else 4) ]} for i in range(2) ] plan = build_dynamic_source_slot_plan(sources, slot_count=1, semantic_cadence=20) timeline = torch.tensor(plan.dense_steps) dist.all_reduce(timeline, op=dist.ReduceOp.MAX) offsets = active_source_schedule_offsets( plan, int(timeline), device=torch.device("cpu"), distributed=True, ) assert offsets == [0, 1, 2, 3, 20, 21, 22, 23] torch.manual_seed(7) model = DistributedDataParallel( torch.nn.Linear(4, 2, bias=True), broadcast_buffers=False, find_unused_parameters=False, ) optimizer = torch.optim.SGD(model.parameters(), lr=0.1) forwards = 1 if rank == 0 else 3 local_max = torch.tensor(forwards, dtype=torch.long) dist.all_reduce(local_max, op=dist.ReduceOp.MAX) loss = None with model.no_sync(): for offset in range(int(local_max.item())): if offset >= forwards: # This rank is padded while the longer rank continues. continue inputs = torch.full((2, 4), float(rank + offset + 1)) value = model(inputs).square().mean() / float(forwards) loss = value if loss is None else loss + value assert loss is not None # One backward after the complete local source, with no graph # truncation or optimizer mutation inside it. loss.backward() average_full_duplex_gradients( model.parameters(), world_size=2, bucket_cap_mb=1 ) optimizer.step() optimizer.zero_grad() flat = torch.cat([parameter.detach().reshape(-1) for parameter in model.parameters()]) gathered = [torch.empty_like(flat) for _ in range(2)] dist.all_gather(gathered, flat) if rank == 0: result_queue.put(bool(torch.equal(gathered[0], gathered[1]))) finally: dist.destroy_process_group() def test_different_source_lengths_use_one_complete_source_collective( tmp_path: Path, ): init_file = tmp_path / "gloo_init" context = mp.get_context("spawn") result_queue = context.Queue() processes = [ context.Process( target=_ddp_worker, args=(rank, str(init_file), result_queue), ) for rank in range(2) ] for process in processes: process.start() for process in processes: # Spawned ranks import the full Transformers/Scipy stack from CPFS # before entering Gloo. Cold FUSE directory scans alone can exceed # 30 seconds even though the collective completes immediately. process.join(timeout=180) assert not process.is_alive(), "DDP gradient synchronization deadlocked" assert process.exitcode == 0 assert result_queue.get(timeout=5) is True