File size: 4,447 Bytes
f972106 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 | from collections import Counter
from psi.data.sampler import TaskMixtureSampler
class _FakeShardAwareDataset:
def __init__(self):
self.calls = 0
def sample_shard_aware_indices(self, batch_size, generator):
self.calls += 1
return list(range(10, 10 + batch_size))
def _dataset_sequence(sampler):
sequence = []
for batch in sampler:
dataset_ids = {dataset_id for dataset_id, _ in batch}
assert len(dataset_ids) == 1
sequence.append(next(iter(dataset_ids)))
return sequence
def test_task_mixture_sampler_batches_are_homogeneous(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
sampler = TaskMixtureSampler(
dataset_lens=[100, 50, 30],
mixture_ratios=[3, 1, 0],
num_samples_per_epoch=64,
batch_size=4,
seed=7,
)
batches = list(sampler)
assert len(batches) == 16
for batch in batches:
assert len(batch) == 4
assert len({dataset_id for dataset_id, _ in batch}) == 1
def test_task_mixture_sampler_respects_batch_ratios(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
sampler = TaskMixtureSampler(
dataset_lens=[100, 50],
mixture_ratios=[3, 1],
num_samples_per_epoch=64,
batch_size=4,
seed=11,
)
counts = Counter(_dataset_sequence(sampler))
assert counts == Counter({0: 12, 1: 4})
def test_task_mixture_sampler_distributed_schedule_matches_global(monkeypatch):
monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "0")
common_kwargs = dict(
dataset_lens=[100, 50, 30],
mixture_ratios=[5, 3, 2],
num_samples_per_epoch=64,
batch_size=4,
seed=19,
)
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
global_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "2")
rank0_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
monkeypatch.setenv("RANK", "1")
monkeypatch.setenv("WORLD_SIZE", "2")
rank1_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
merged_sequence = []
for batch_idx in range(len(global_sequence)):
local_idx = batch_idx // 2
merged_sequence.append(rank0_sequence[local_idx] if batch_idx % 2 == 0 else rank1_sequence[local_idx])
assert merged_sequence == global_sequence
def test_task_mixture_sampler_can_sync_tasks_across_ranks(monkeypatch):
common_kwargs = dict(
dataset_lens=[100, 50, 30],
mixture_ratios=[5, 3, 2],
num_samples_per_epoch=64,
batch_size=4,
seed=23,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "1")
monkeypatch.setenv("WORLD_SIZE", "2")
monkeypatch.setenv("RANK", "0")
rank0_batches = list(TaskMixtureSampler(**common_kwargs))
rank0_sequence = _dataset_sequence(rank0_batches)
monkeypatch.setenv("RANK", "1")
rank1_batches = list(TaskMixtureSampler(**common_kwargs))
rank1_sequence = _dataset_sequence(rank1_batches)
assert rank0_sequence == rank1_sequence
assert all(
{sample_id for _, sample_id in batch0} != {sample_id for _, sample_id in batch1}
for batch0, batch1 in zip(rank0_batches, rank1_batches)
)
def test_task_mixture_sampler_uses_shard_aware_indices_only_when_enabled(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
dataset = _FakeShardAwareDataset()
common_kwargs = dict(
dataset_lens=[100],
datasets=[dataset],
mixture_ratios=[1],
num_samples_per_epoch=16,
batch_size=4,
seed=31,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
default_batches = list(TaskMixtureSampler(**common_kwargs))
assert dataset.calls == 0
assert any(sample_id < 10 or sample_id >= 14 for batch in default_batches for _, sample_id in batch)
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
shard_batches = list(TaskMixtureSampler(**common_kwargs))
assert dataset.calls == len(shard_batches)
assert all([sample_id for _, sample_id in batch] == [10, 11, 12, 13] for batch in shard_batches)
|