File size: 4,694 Bytes
aa02c28
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
from __future__ import annotations

import multiprocessing as mp
from pathlib import Path

import torch
import torch.distributed as dist
from torch.nn.parallel import DistributedDataParallel

from psi.trainers.posttrain_full_duplex import (
    _all_reduce_sum_copy,
    average_full_duplex_gradients,
    build_dynamic_source_slot_plan,
    active_source_schedule_offsets,
)


def test_metric_collective_does_not_mutate_rank_local_denominator(monkeypatch):
    local_count = torch.tensor(7.0, requires_grad=True)

    def fake_all_reduce(value, op):
        assert op == dist.ReduceOp.SUM
        value.mul_(8.0)

    monkeypatch.setattr(dist, "all_reduce", fake_all_reduce)
    global_count = _all_reduce_sum_copy(local_count, distributed=True)

    assert global_count.item() == 56.0
    assert local_count.item() == 7.0
    assert global_count.data_ptr() != local_count.data_ptr()
    assert global_count.requires_grad is False

    metric_row = torch.stack(
        [torch.tensor(5.0), local_count.detach()]
    )
    dist.all_reduce(metric_row, op=dist.ReduceOp.SUM)
    assert torch.isclose(
        metric_row[0] / metric_row[1],
        torch.tensor(5.0 / 7.0),
    )


def _ddp_worker(rank: int, init_file: str, result_queue) -> None:
    dist.init_process_group(
        "gloo",
        init_method=f"file://{init_file}",
        rank=rank,
        world_size=2,
    )
    try:
        # Rank 0's local timeline has a gap at 2,3; rank 1 still has real
        # windows there. Both must keep the union and drop only the shared
        # gap 4..19, including when their local timelines have different ends.
        sources = [
            {"source_id": str(i), "steps": [
                {"update_semantic_cache": j == 0}
                for j in range(2 if rank == 0 else 4)
            ]} for i in range(2)
        ]
        plan = build_dynamic_source_slot_plan(sources, slot_count=1, semantic_cadence=20)
        timeline = torch.tensor(plan.dense_steps)
        dist.all_reduce(timeline, op=dist.ReduceOp.MAX)
        offsets = active_source_schedule_offsets(
            plan, int(timeline), device=torch.device("cpu"), distributed=True,
        )
        assert offsets == [0, 1, 2, 3, 20, 21, 22, 23]
        torch.manual_seed(7)
        model = DistributedDataParallel(
            torch.nn.Linear(4, 2, bias=True),
            broadcast_buffers=False,
            find_unused_parameters=False,
        )
        optimizer = torch.optim.SGD(model.parameters(), lr=0.1)
        forwards = 1 if rank == 0 else 3
        local_max = torch.tensor(forwards, dtype=torch.long)
        dist.all_reduce(local_max, op=dist.ReduceOp.MAX)
        loss = None
        with model.no_sync():
            for offset in range(int(local_max.item())):
                if offset >= forwards:
                    # This rank is padded while the longer rank continues.
                    continue
                inputs = torch.full((2, 4), float(rank + offset + 1))
                value = model(inputs).square().mean() / float(forwards)
                loss = value if loss is None else loss + value
            assert loss is not None
            # One backward after the complete local source, with no graph
            # truncation or optimizer mutation inside it.
            loss.backward()
        average_full_duplex_gradients(
            model.parameters(), world_size=2, bucket_cap_mb=1
        )
        optimizer.step()
        optimizer.zero_grad()

        flat = torch.cat([parameter.detach().reshape(-1) for parameter in model.parameters()])
        gathered = [torch.empty_like(flat) for _ in range(2)]
        dist.all_gather(gathered, flat)
        if rank == 0:
            result_queue.put(bool(torch.equal(gathered[0], gathered[1])))
    finally:
        dist.destroy_process_group()


def test_different_source_lengths_use_one_complete_source_collective(
    tmp_path: Path,
):
    init_file = tmp_path / "gloo_init"
    context = mp.get_context("spawn")
    result_queue = context.Queue()
    processes = [
        context.Process(
            target=_ddp_worker,
            args=(rank, str(init_file), result_queue),
        )
        for rank in range(2)
    ]
    for process in processes:
        process.start()
    for process in processes:
        # Spawned ranks import the full Transformers/Scipy stack from CPFS
        # before entering Gloo.  Cold FUSE directory scans alone can exceed
        # 30 seconds even though the collective completes immediately.
        process.join(timeout=180)
        assert not process.is_alive(), "DDP gradient synchronization deadlocked"
        assert process.exitcode == 0
    assert result_queue.get(timeout=5) is True