File size: 7,830 Bytes
02626a9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 | from collections import Counter
from psi.data.sampler import TaskMixtureSampler
from psi.data.seed.omni_oss_sampler import OssShardBatch
class _FakeShardAwareDataset:
def __init__(self):
self.calls = 0
def sample_shard_aware_indices(self, batch_size, generator):
self.calls += 1
return list(range(10, 10 + batch_size))
def _dataset_sequence(sampler):
sequence = []
for batch in sampler:
dataset_ids = {item[0] for item in batch}
assert len(dataset_ids) == 1
sequence.append(next(iter(dataset_ids)))
return sequence
def test_task_mixture_sampler_batches_are_homogeneous(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
sampler = TaskMixtureSampler(
dataset_lens=[100, 50, 30],
mixture_ratios=[3, 1, 0],
num_samples_per_epoch=64,
batch_size=4,
seed=7,
)
batches = list(sampler)
assert len(batches) == 16
for batch in batches:
assert len(batch) == 4
assert len({dataset_id for dataset_id, _ in batch}) == 1
def test_task_mixture_sampler_respects_batch_ratios(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
sampler = TaskMixtureSampler(
dataset_lens=[100, 50],
mixture_ratios=[3, 1],
num_samples_per_epoch=64,
batch_size=4,
seed=11,
)
counts = Counter(_dataset_sequence(sampler))
assert counts == Counter({0: 12, 1: 4})
def test_task_mixture_sampler_distributed_schedule_matches_global(monkeypatch):
monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "0")
common_kwargs = dict(
dataset_lens=[100, 50, 30],
mixture_ratios=[5, 3, 2],
num_samples_per_epoch=64,
batch_size=4,
seed=19,
)
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
global_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "2")
rank0_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
monkeypatch.setenv("RANK", "1")
monkeypatch.setenv("WORLD_SIZE", "2")
rank1_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
merged_sequence = []
for batch_idx in range(len(global_sequence)):
local_idx = batch_idx // 2
merged_sequence.append(rank0_sequence[local_idx] if batch_idx % 2 == 0 else rank1_sequence[local_idx])
assert merged_sequence == global_sequence
def test_task_mixture_sampler_can_sync_tasks_across_ranks(monkeypatch):
common_kwargs = dict(
dataset_lens=[100, 50, 30],
mixture_ratios=[5, 3, 2],
num_samples_per_epoch=64,
batch_size=4,
seed=23,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "1")
monkeypatch.setenv("WORLD_SIZE", "2")
monkeypatch.setenv("RANK", "0")
rank0_batches = list(TaskMixtureSampler(**common_kwargs))
rank0_sequence = _dataset_sequence(rank0_batches)
monkeypatch.setenv("RANK", "1")
rank1_batches = list(TaskMixtureSampler(**common_kwargs))
rank1_sequence = _dataset_sequence(rank1_batches)
assert rank0_sequence == rank1_sequence
assert all(
{sample_id for _, sample_id in batch0} != {sample_id for _, sample_id in batch1}
for batch0, batch1 in zip(rank0_batches, rank1_batches)
)
def test_task_mixture_sampler_balances_equal_tasks_across_ranks(monkeypatch):
common_kwargs = dict(
dataset_lens=[100, 100, 100, 100],
mixture_ratios=[1, 1, 1, 1],
num_samples_per_epoch=64,
batch_size=4,
seed=23,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
monkeypatch.setenv("WORLD_SIZE", "8")
rank_sequences = []
for rank in range(8):
monkeypatch.setenv("RANK", str(rank))
batches = list(TaskMixtureSampler(**common_kwargs))
assert batches
task_ids = [{item[0] for item in batch} for batch in batches]
assert all(len(ids) == 1 for ids in task_ids)
rank_sequences.append([next(iter(ids)) for ids in task_ids])
# Every local step has exactly two ranks assigned to each enabled task.
for local_step in range(len(rank_sequences[0])):
assert Counter(sequence[local_step] for sequence in rank_sequences) == Counter(
{0: 2, 1: 2, 2: 2, 3: 2}
)
def test_task_mixture_sampler_assigns_exact_weighted_rank_counts(monkeypatch):
common_kwargs = dict(
dataset_lens=[100, 100, 100, 100],
mixture_ratios=[3, 1, 3, 1],
num_samples_per_epoch=64,
batch_size=4,
seed=29,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
monkeypatch.setenv("WORLD_SIZE", "8")
rank_sequences = []
for rank in range(8):
monkeypatch.setenv("RANK", str(rank))
batches = list(TaskMixtureSampler(**common_kwargs))
assert batches
rank_sequences.append(_dataset_sequence(batches))
for local_step in range(len(rank_sequences[0])):
assert Counter(sequence[local_step] for sequence in rank_sequences) == Counter(
{0: 3, 1: 1, 2: 3, 3: 1}
)
def test_task_mixture_sampler_uses_shard_aware_indices_only_when_enabled(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
dataset = _FakeShardAwareDataset()
common_kwargs = dict(
dataset_lens=[100],
datasets=[dataset],
mixture_ratios=[1],
num_samples_per_epoch=16,
batch_size=4,
seed=31,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
default_batches = list(TaskMixtureSampler(**common_kwargs))
assert dataset.calls == 0
assert any(sample_id < 10 or sample_id >= 14 for batch in default_batches for _, sample_id in batch)
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
shard_batches = list(TaskMixtureSampler(**common_kwargs))
assert dataset.calls == len(shard_batches)
assert all([sample_id for _, sample_id in batch] == [10, 11, 12, 13] for batch in shard_batches)
def test_task_mixture_sampler_emits_oss_hints_and_worker_lanes(monkeypatch):
class _FakeOssDataset:
def __init__(self):
self.lanes = []
def sample_shard_aware_indices(self, batch_size, generator, lane_id=0):
self.lanes.append(lane_id)
return OssShardBatch(
indices=list(range(lane_id * 100, lane_id * 100 + batch_size)),
current_shard=f"/oss/current-{lane_id}.safetensors",
next_shard=f"/oss/next-{lane_id}.safetensors",
shard_id=lane_id,
)
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_BLOCK_BATCHES", "4")
monkeypatch.setenv("PSI_TASK_MIXTURE_RANK_LOCAL_BLOCKS", "1")
monkeypatch.setenv("PSI_OMNI_OSS_WORKER_LANES", "2")
dataset = _FakeOssDataset()
sampler = TaskMixtureSampler(
dataset_lens=[1000],
datasets=[dataset],
mixture_ratios=[1],
num_samples_per_epoch=32,
batch_size=4,
seed=37,
)
batches = list(sampler)
assert dataset.lanes == [0, 1, 0, 1, 0, 1, 0, 1]
assert all(len(item) == 4 for batch in batches for item in batch)
assert all(item[2] == f"/oss/current-{batch_id % 2}.safetensors" for batch_id, batch in enumerate(batches) for item in batch)
|