File size: 4,447 Bytes
f972106
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
from collections import Counter

from psi.data.sampler import TaskMixtureSampler


class _FakeShardAwareDataset:
    def __init__(self):
        self.calls = 0

    def sample_shard_aware_indices(self, batch_size, generator):
        self.calls += 1
        return list(range(10, 10 + batch_size))


def _dataset_sequence(sampler):
    sequence = []
    for batch in sampler:
        dataset_ids = {dataset_id for dataset_id, _ in batch}
        assert len(dataset_ids) == 1
        sequence.append(next(iter(dataset_ids)))
    return sequence


def test_task_mixture_sampler_batches_are_homogeneous(monkeypatch):
    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "1")
    monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")

    sampler = TaskMixtureSampler(
        dataset_lens=[100, 50, 30],
        mixture_ratios=[3, 1, 0],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=7,
    )

    batches = list(sampler)
    assert len(batches) == 16
    for batch in batches:
        assert len(batch) == 4
        assert len({dataset_id for dataset_id, _ in batch}) == 1


def test_task_mixture_sampler_respects_batch_ratios(monkeypatch):
    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "1")
    monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")

    sampler = TaskMixtureSampler(
        dataset_lens=[100, 50],
        mixture_ratios=[3, 1],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=11,
    )

    counts = Counter(_dataset_sequence(sampler))
    assert counts == Counter({0: 12, 1: 4})


def test_task_mixture_sampler_distributed_schedule_matches_global(monkeypatch):
    monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "0")
    common_kwargs = dict(
        dataset_lens=[100, 50, 30],
        mixture_ratios=[5, 3, 2],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=19,
    )

    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "1")
    global_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))

    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "2")
    rank0_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))

    monkeypatch.setenv("RANK", "1")
    monkeypatch.setenv("WORLD_SIZE", "2")
    rank1_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))

    merged_sequence = []
    for batch_idx in range(len(global_sequence)):
        local_idx = batch_idx // 2
        merged_sequence.append(rank0_sequence[local_idx] if batch_idx % 2 == 0 else rank1_sequence[local_idx])

    assert merged_sequence == global_sequence


def test_task_mixture_sampler_can_sync_tasks_across_ranks(monkeypatch):
    common_kwargs = dict(
        dataset_lens=[100, 50, 30],
        mixture_ratios=[5, 3, 2],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=23,
    )

    monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "1")
    monkeypatch.setenv("WORLD_SIZE", "2")

    monkeypatch.setenv("RANK", "0")
    rank0_batches = list(TaskMixtureSampler(**common_kwargs))
    rank0_sequence = _dataset_sequence(rank0_batches)

    monkeypatch.setenv("RANK", "1")
    rank1_batches = list(TaskMixtureSampler(**common_kwargs))
    rank1_sequence = _dataset_sequence(rank1_batches)

    assert rank0_sequence == rank1_sequence
    assert all(
        {sample_id for _, sample_id in batch0} != {sample_id for _, sample_id in batch1}
        for batch0, batch1 in zip(rank0_batches, rank1_batches)
    )


def test_task_mixture_sampler_uses_shard_aware_indices_only_when_enabled(monkeypatch):
    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "1")
    dataset = _FakeShardAwareDataset()
    common_kwargs = dict(
        dataset_lens=[100],
        datasets=[dataset],
        mixture_ratios=[1],
        num_samples_per_epoch=16,
        batch_size=4,
        seed=31,
    )

    monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
    default_batches = list(TaskMixtureSampler(**common_kwargs))
    assert dataset.calls == 0
    assert any(sample_id < 10 or sample_id >= 14 for batch in default_batches for _, sample_id in batch)

    monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
    shard_batches = list(TaskMixtureSampler(**common_kwargs))
    assert dataset.calls == len(shard_batches)
    assert all([sample_id for _, sample_id in batch] == [10, 11, 12, 13] for batch in shard_batches)