File size: 7,830 Bytes
02626a9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
from collections import Counter

from psi.data.sampler import TaskMixtureSampler
from psi.data.seed.omni_oss_sampler import OssShardBatch


class _FakeShardAwareDataset:
    def __init__(self):
        self.calls = 0

    def sample_shard_aware_indices(self, batch_size, generator):
        self.calls += 1
        return list(range(10, 10 + batch_size))


def _dataset_sequence(sampler):
    sequence = []
    for batch in sampler:
        dataset_ids = {item[0] for item in batch}
        assert len(dataset_ids) == 1
        sequence.append(next(iter(dataset_ids)))
    return sequence


def test_task_mixture_sampler_batches_are_homogeneous(monkeypatch):
    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "1")
    monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")

    sampler = TaskMixtureSampler(
        dataset_lens=[100, 50, 30],
        mixture_ratios=[3, 1, 0],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=7,
    )

    batches = list(sampler)
    assert len(batches) == 16
    for batch in batches:
        assert len(batch) == 4
        assert len({dataset_id for dataset_id, _ in batch}) == 1


def test_task_mixture_sampler_respects_batch_ratios(monkeypatch):
    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "1")
    monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")

    sampler = TaskMixtureSampler(
        dataset_lens=[100, 50],
        mixture_ratios=[3, 1],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=11,
    )

    counts = Counter(_dataset_sequence(sampler))
    assert counts == Counter({0: 12, 1: 4})


def test_task_mixture_sampler_distributed_schedule_matches_global(monkeypatch):
    monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "0")
    common_kwargs = dict(
        dataset_lens=[100, 50, 30],
        mixture_ratios=[5, 3, 2],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=19,
    )

    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "1")
    global_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))

    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "2")
    rank0_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))

    monkeypatch.setenv("RANK", "1")
    monkeypatch.setenv("WORLD_SIZE", "2")
    rank1_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))

    merged_sequence = []
    for batch_idx in range(len(global_sequence)):
        local_idx = batch_idx // 2
        merged_sequence.append(rank0_sequence[local_idx] if batch_idx % 2 == 0 else rank1_sequence[local_idx])

    assert merged_sequence == global_sequence


def test_task_mixture_sampler_can_sync_tasks_across_ranks(monkeypatch):
    common_kwargs = dict(
        dataset_lens=[100, 50, 30],
        mixture_ratios=[5, 3, 2],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=23,
    )

    monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "1")
    monkeypatch.setenv("WORLD_SIZE", "2")

    monkeypatch.setenv("RANK", "0")
    rank0_batches = list(TaskMixtureSampler(**common_kwargs))
    rank0_sequence = _dataset_sequence(rank0_batches)

    monkeypatch.setenv("RANK", "1")
    rank1_batches = list(TaskMixtureSampler(**common_kwargs))
    rank1_sequence = _dataset_sequence(rank1_batches)

    assert rank0_sequence == rank1_sequence
    assert all(
        {sample_id for _, sample_id in batch0} != {sample_id for _, sample_id in batch1}
        for batch0, batch1 in zip(rank0_batches, rank1_batches)
    )


def test_task_mixture_sampler_balances_equal_tasks_across_ranks(monkeypatch):
    common_kwargs = dict(
        dataset_lens=[100, 100, 100, 100],
        mixture_ratios=[1, 1, 1, 1],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=23,
    )
    monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1")
    monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
    monkeypatch.setenv("WORLD_SIZE", "8")

    rank_sequences = []
    for rank in range(8):
        monkeypatch.setenv("RANK", str(rank))
        batches = list(TaskMixtureSampler(**common_kwargs))
        assert batches
        task_ids = [{item[0] for item in batch} for batch in batches]
        assert all(len(ids) == 1 for ids in task_ids)
        rank_sequences.append([next(iter(ids)) for ids in task_ids])

    # Every local step has exactly two ranks assigned to each enabled task.
    for local_step in range(len(rank_sequences[0])):
        assert Counter(sequence[local_step] for sequence in rank_sequences) == Counter(
            {0: 2, 1: 2, 2: 2, 3: 2}
        )


def test_task_mixture_sampler_assigns_exact_weighted_rank_counts(monkeypatch):
    common_kwargs = dict(
        dataset_lens=[100, 100, 100, 100],
        mixture_ratios=[3, 1, 3, 1],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=29,
    )
    monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1")
    monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
    monkeypatch.setenv("WORLD_SIZE", "8")

    rank_sequences = []
    for rank in range(8):
        monkeypatch.setenv("RANK", str(rank))
        batches = list(TaskMixtureSampler(**common_kwargs))
        assert batches
        rank_sequences.append(_dataset_sequence(batches))

    for local_step in range(len(rank_sequences[0])):
        assert Counter(sequence[local_step] for sequence in rank_sequences) == Counter(
            {0: 3, 1: 1, 2: 3, 3: 1}
        )


def test_task_mixture_sampler_uses_shard_aware_indices_only_when_enabled(monkeypatch):
    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "1")
    dataset = _FakeShardAwareDataset()
    common_kwargs = dict(
        dataset_lens=[100],
        datasets=[dataset],
        mixture_ratios=[1],
        num_samples_per_epoch=16,
        batch_size=4,
        seed=31,
    )

    monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
    default_batches = list(TaskMixtureSampler(**common_kwargs))
    assert dataset.calls == 0
    assert any(sample_id < 10 or sample_id >= 14 for batch in default_batches for _, sample_id in batch)

    monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
    shard_batches = list(TaskMixtureSampler(**common_kwargs))
    assert dataset.calls == len(shard_batches)
    assert all([sample_id for _, sample_id in batch] == [10, 11, 12, 13] for batch in shard_batches)


def test_task_mixture_sampler_emits_oss_hints_and_worker_lanes(monkeypatch):
    class _FakeOssDataset:
        def __init__(self):
            self.lanes = []

        def sample_shard_aware_indices(self, batch_size, generator, lane_id=0):
            self.lanes.append(lane_id)
            return OssShardBatch(
                indices=list(range(lane_id * 100, lane_id * 100 + batch_size)),
                current_shard=f"/oss/current-{lane_id}.safetensors",
                next_shard=f"/oss/next-{lane_id}.safetensors",
                shard_id=lane_id,
            )

    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "1")
    monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
    monkeypatch.setenv("PSI_TASK_MIXTURE_BLOCK_BATCHES", "4")
    monkeypatch.setenv("PSI_TASK_MIXTURE_RANK_LOCAL_BLOCKS", "1")
    monkeypatch.setenv("PSI_OMNI_OSS_WORKER_LANES", "2")
    dataset = _FakeOssDataset()
    sampler = TaskMixtureSampler(
        dataset_lens=[1000],
        datasets=[dataset],
        mixture_ratios=[1],
        num_samples_per_epoch=32,
        batch_size=4,
        seed=37,
    )

    batches = list(sampler)
    assert dataset.lanes == [0, 1, 0, 1, 0, 1, 0, 1]
    assert all(len(item) == 4 for batch in batches for item in batch)
    assert all(item[2] == f"/oss/current-{batch_id % 2}.safetensors" for batch_id, batch in enumerate(batches) for item in batch)