Download tests/test_full_duplex_ddp_sync.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 4.69 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/tests/test_full_duplex_ddp_sync.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/tests/test_full_duplex_ddp_sync.py
-
curl -L -o test_full_duplex_ddp_sync.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/tests/test_full_duplex_ddp_sync.py
4.69 kB
| from __future__ import annotations | |
| import multiprocessing as mp | |
| from pathlib import Path | |
| import torch | |
| import torch.distributed as dist | |
| from torch.nn.parallel import DistributedDataParallel | |
| from psi.trainers.posttrain_full_duplex import ( | |
| _all_reduce_sum_copy, | |
| average_full_duplex_gradients, | |
| build_dynamic_source_slot_plan, | |
| active_source_schedule_offsets, | |
| ) | |
| def test_metric_collective_does_not_mutate_rank_local_denominator(monkeypatch): | |
| local_count = torch.tensor(7.0, requires_grad=True) | |
| def fake_all_reduce(value, op): | |
| assert op == dist.ReduceOp.SUM | |
| value.mul_(8.0) | |
| monkeypatch.setattr(dist, "all_reduce", fake_all_reduce) | |
| global_count = _all_reduce_sum_copy(local_count, distributed=True) | |
| assert global_count.item() == 56.0 | |
| assert local_count.item() == 7.0 | |
| assert global_count.data_ptr() != local_count.data_ptr() | |
| assert global_count.requires_grad is False | |
| metric_row = torch.stack( | |
| [torch.tensor(5.0), local_count.detach()] | |
| ) | |
| dist.all_reduce(metric_row, op=dist.ReduceOp.SUM) | |
| assert torch.isclose( | |
| metric_row[0] / metric_row[1], | |
| torch.tensor(5.0 / 7.0), | |
| ) | |
| def _ddp_worker(rank: int, init_file: str, result_queue) -> None: | |
| dist.init_process_group( | |
| "gloo", | |
| init_method=f"file://{init_file}", | |
| rank=rank, | |
| world_size=2, | |
| ) | |
| try: | |
| # Rank 0's local timeline has a gap at 2,3; rank 1 still has real | |
| # windows there. Both must keep the union and drop only the shared | |
| # gap 4..19, including when their local timelines have different ends. | |
| sources = [ | |
| {"source_id": str(i), "steps": [ | |
| {"update_semantic_cache": j == 0} | |
| for j in range(2 if rank == 0 else 4) | |
| ]} for i in range(2) | |
| ] | |
| plan = build_dynamic_source_slot_plan(sources, slot_count=1, semantic_cadence=20) | |
| timeline = torch.tensor(plan.dense_steps) | |
| dist.all_reduce(timeline, op=dist.ReduceOp.MAX) | |
| offsets = active_source_schedule_offsets( | |
| plan, int(timeline), device=torch.device("cpu"), distributed=True, | |
| ) | |
| assert offsets == [0, 1, 2, 3, 20, 21, 22, 23] | |
| torch.manual_seed(7) | |
| model = DistributedDataParallel( | |
| torch.nn.Linear(4, 2, bias=True), | |
| broadcast_buffers=False, | |
| find_unused_parameters=False, | |
| ) | |
| optimizer = torch.optim.SGD(model.parameters(), lr=0.1) | |
| forwards = 1 if rank == 0 else 3 | |
| local_max = torch.tensor(forwards, dtype=torch.long) | |
| dist.all_reduce(local_max, op=dist.ReduceOp.MAX) | |
| loss = None | |
| with model.no_sync(): | |
| for offset in range(int(local_max.item())): | |
| if offset >= forwards: | |
| # This rank is padded while the longer rank continues. | |
| continue | |
| inputs = torch.full((2, 4), float(rank + offset + 1)) | |
| value = model(inputs).square().mean() / float(forwards) | |
| loss = value if loss is None else loss + value | |
| assert loss is not None | |
| # One backward after the complete local source, with no graph | |
| # truncation or optimizer mutation inside it. | |
| loss.backward() | |
| average_full_duplex_gradients( | |
| model.parameters(), world_size=2, bucket_cap_mb=1 | |
| ) | |
| optimizer.step() | |
| optimizer.zero_grad() | |
| flat = torch.cat([parameter.detach().reshape(-1) for parameter in model.parameters()]) | |
| gathered = [torch.empty_like(flat) for _ in range(2)] | |
| dist.all_gather(gathered, flat) | |
| if rank == 0: | |
| result_queue.put(bool(torch.equal(gathered[0], gathered[1]))) | |
| finally: | |
| dist.destroy_process_group() | |
| def test_different_source_lengths_use_one_complete_source_collective( | |
| tmp_path: Path, | |
| ): | |
| init_file = tmp_path / "gloo_init" | |
| context = mp.get_context("spawn") | |
| result_queue = context.Queue() | |
| processes = [ | |
| context.Process( | |
| target=_ddp_worker, | |
| args=(rank, str(init_file), result_queue), | |
| ) | |
| for rank in range(2) | |
| ] | |
| for process in processes: | |
| process.start() | |
| for process in processes: | |
| # Spawned ranks import the full Transformers/Scipy stack from CPFS | |
| # before entering Gloo. Cold FUSE directory scans alone can exceed | |
| # 30 seconds even though the collective completes immediately. | |
| process.join(timeout=180) | |
| assert not process.is_alive(), "DDP gradient synchronization deadlocked" | |
| assert process.exitcode == 0 | |
| assert result_queue.get(timeout=5) is True | |