code-Psi0-yth-v23 / src /psi /data /sampler_test.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
f972106 verified
Raw History Blame Contribute Delete
4.45 kB
from collections import Counter
from psi.data.sampler import TaskMixtureSampler
class _FakeShardAwareDataset:
def __init__(self):
self.calls = 0
def sample_shard_aware_indices(self, batch_size, generator):
self.calls += 1
return list(range(10, 10 + batch_size))
def _dataset_sequence(sampler):
sequence = []
for batch in sampler:
dataset_ids = {dataset_id for dataset_id, _ in batch}
assert len(dataset_ids) == 1
sequence.append(next(iter(dataset_ids)))
return sequence
def test_task_mixture_sampler_batches_are_homogeneous(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
sampler = TaskMixtureSampler(
dataset_lens=[100, 50, 30],
mixture_ratios=[3, 1, 0],
num_samples_per_epoch=64,
batch_size=4,
seed=7,
)
batches = list(sampler)
assert len(batches) == 16
for batch in batches:
assert len(batch) == 4
assert len({dataset_id for dataset_id, _ in batch}) == 1
def test_task_mixture_sampler_respects_batch_ratios(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
sampler = TaskMixtureSampler(
dataset_lens=[100, 50],
mixture_ratios=[3, 1],
num_samples_per_epoch=64,
batch_size=4,
seed=11,
)
counts = Counter(_dataset_sequence(sampler))
assert counts == Counter({0: 12, 1: 4})
def test_task_mixture_sampler_distributed_schedule_matches_global(monkeypatch):
monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "0")
common_kwargs = dict(
dataset_lens=[100, 50, 30],
mixture_ratios=[5, 3, 2],
num_samples_per_epoch=64,
batch_size=4,
seed=19,
)
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
global_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "2")
rank0_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
monkeypatch.setenv("RANK", "1")
monkeypatch.setenv("WORLD_SIZE", "2")
rank1_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
merged_sequence = []
for batch_idx in range(len(global_sequence)):
local_idx = batch_idx // 2
merged_sequence.append(rank0_sequence[local_idx] if batch_idx % 2 == 0 else rank1_sequence[local_idx])
assert merged_sequence == global_sequence
def test_task_mixture_sampler_can_sync_tasks_across_ranks(monkeypatch):
common_kwargs = dict(
dataset_lens=[100, 50, 30],
mixture_ratios=[5, 3, 2],
num_samples_per_epoch=64,
batch_size=4,
seed=23,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "1")
monkeypatch.setenv("WORLD_SIZE", "2")
monkeypatch.setenv("RANK", "0")
rank0_batches = list(TaskMixtureSampler(**common_kwargs))
rank0_sequence = _dataset_sequence(rank0_batches)
monkeypatch.setenv("RANK", "1")
rank1_batches = list(TaskMixtureSampler(**common_kwargs))
rank1_sequence = _dataset_sequence(rank1_batches)
assert rank0_sequence == rank1_sequence
assert all(
{sample_id for _, sample_id in batch0} != {sample_id for _, sample_id in batch1}
for batch0, batch1 in zip(rank0_batches, rank1_batches)
)
def test_task_mixture_sampler_uses_shard_aware_indices_only_when_enabled(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
dataset = _FakeShardAwareDataset()
common_kwargs = dict(
dataset_lens=[100],
datasets=[dataset],
mixture_ratios=[1],
num_samples_per_epoch=16,
batch_size=4,
seed=31,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
default_batches = list(TaskMixtureSampler(**common_kwargs))
assert dataset.calls == 0
assert any(sample_id < 10 or sample_id >= 14 for batch in default_batches for _, sample_id in batch)
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
shard_batches = list(TaskMixtureSampler(**common_kwargs))
assert dataset.calls == len(shard_batches)
assert all([sample_id for _, sample_id in batch] == [10, 11, 12, 13] for batch in shard_batches)