File size: 2,248 Bytes
05481f9
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
from collections import Counter

from psi.data.sampler import TaskMixtureSampler


def _dataset_sequence(sampler):
    sequence = []
    for batch in sampler:
        dataset_ids = {dataset_id for dataset_id, _ in batch}
        assert len(dataset_ids) == 1
        sequence.append(next(iter(dataset_ids)))
    return sequence


def test_task_mixture_sampler_batches_are_homogeneous(monkeypatch):
    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "1")

    sampler = TaskMixtureSampler(
        dataset_lens=[100, 50, 30],
        mixture_ratios=[3, 1, 0],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=7,
    )

    batches = list(sampler)
    assert len(batches) == 16
    for batch in batches:
        assert len(batch) == 4
        assert len({dataset_id for dataset_id, _ in batch}) == 1


def test_task_mixture_sampler_respects_batch_ratios(monkeypatch):
    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "1")

    sampler = TaskMixtureSampler(
        dataset_lens=[100, 50],
        mixture_ratios=[3, 1],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=11,
    )

    counts = Counter(_dataset_sequence(sampler))
    assert counts == Counter({0: 12, 1: 4})


def test_task_mixture_sampler_distributed_schedule_matches_global(monkeypatch):
    common_kwargs = dict(
        dataset_lens=[100, 50, 30],
        mixture_ratios=[5, 3, 2],
        num_samples_per_epoch=64,
        batch_size=4,
        seed=19,
    )

    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "1")
    global_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))

    monkeypatch.setenv("RANK", "0")
    monkeypatch.setenv("WORLD_SIZE", "2")
    rank0_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))

    monkeypatch.setenv("RANK", "1")
    monkeypatch.setenv("WORLD_SIZE", "2")
    rank1_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))

    merged_sequence = []
    for batch_idx in range(len(global_sequence)):
        local_idx = batch_idx // 2
        merged_sequence.append(rank0_sequence[local_idx] if batch_idx % 2 == 0 else rank1_sequence[local_idx])

    assert merged_sequence == global_sequence