code-Psi0-yth-v24 / src /psi /data /sampler_test.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
ffa47ee verified
Raw History Blame Contribute Delete
6.97 kB
from collections import Counter
from psi.data.sampler import TaskMixtureSampler
from psi.data.seed.omni_oss_sampler import OssShardBatch
class _FakeShardAwareDataset:
def __init__(self):
self.calls = 0
def sample_shard_aware_indices(self, batch_size, generator):
self.calls += 1
return list(range(10, 10 + batch_size))
def _dataset_sequence(sampler):
sequence = []
for batch in sampler:
dataset_ids = {item[0] for item in batch}
assert len(dataset_ids) == 1
sequence.append(next(iter(dataset_ids)))
return sequence
def test_task_mixture_sampler_batches_are_homogeneous(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
sampler = TaskMixtureSampler(
dataset_lens=[100, 50, 30],
mixture_ratios=[3, 1, 0],
num_samples_per_epoch=64,
batch_size=4,
seed=7,
)
batches = list(sampler)
assert len(batches) == 16
for batch in batches:
assert len(batch) == 4
assert len({dataset_id for dataset_id, _ in batch}) == 1
def test_task_mixture_sampler_respects_batch_ratios(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
sampler = TaskMixtureSampler(
dataset_lens=[100, 50],
mixture_ratios=[3, 1],
num_samples_per_epoch=64,
batch_size=4,
seed=11,
)
counts = Counter(_dataset_sequence(sampler))
assert counts == Counter({0: 12, 1: 4})
def test_task_mixture_sampler_distributed_schedule_matches_global(monkeypatch):
monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "0")
common_kwargs = dict(
dataset_lens=[100, 50, 30],
mixture_ratios=[5, 3, 2],
num_samples_per_epoch=64,
batch_size=4,
seed=19,
)
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
global_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "2")
rank0_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
monkeypatch.setenv("RANK", "1")
monkeypatch.setenv("WORLD_SIZE", "2")
rank1_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
merged_sequence = []
for batch_idx in range(len(global_sequence)):
local_idx = batch_idx // 2
merged_sequence.append(rank0_sequence[local_idx] if batch_idx % 2 == 0 else rank1_sequence[local_idx])
assert merged_sequence == global_sequence
def test_task_mixture_sampler_can_sync_tasks_across_ranks(monkeypatch):
common_kwargs = dict(
dataset_lens=[100, 50, 30],
mixture_ratios=[5, 3, 2],
num_samples_per_epoch=64,
batch_size=4,
seed=23,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "1")
monkeypatch.setenv("WORLD_SIZE", "2")
monkeypatch.setenv("RANK", "0")
rank0_batches = list(TaskMixtureSampler(**common_kwargs))
rank0_sequence = _dataset_sequence(rank0_batches)
monkeypatch.setenv("RANK", "1")
rank1_batches = list(TaskMixtureSampler(**common_kwargs))
rank1_sequence = _dataset_sequence(rank1_batches)
assert rank0_sequence == rank1_sequence
assert all(
{sample_id for _, sample_id in batch0} != {sample_id for _, sample_id in batch1}
for batch0, batch1 in zip(rank0_batches, rank1_batches)
)
def test_task_mixture_sampler_balances_equal_tasks_across_ranks(monkeypatch):
common_kwargs = dict(
dataset_lens=[100, 100, 100, 100],
mixture_ratios=[1, 1, 1, 1],
num_samples_per_epoch=64,
batch_size=4,
seed=23,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
monkeypatch.setenv("WORLD_SIZE", "8")
rank_sequences = []
for rank in range(8):
monkeypatch.setenv("RANK", str(rank))
batches = list(TaskMixtureSampler(**common_kwargs))
assert batches
task_ids = [{item[0] for item in batch} for batch in batches]
assert all(len(ids) == 1 for ids in task_ids)
rank_sequences.append([next(iter(ids)) for ids in task_ids])
# Every local step has exactly two ranks assigned to each enabled task.
for local_step in range(len(rank_sequences[0])):
assert Counter(sequence[local_step] for sequence in rank_sequences) == Counter(
{0: 2, 1: 2, 2: 2, 3: 2}
)
def test_task_mixture_sampler_uses_shard_aware_indices_only_when_enabled(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
dataset = _FakeShardAwareDataset()
common_kwargs = dict(
dataset_lens=[100],
datasets=[dataset],
mixture_ratios=[1],
num_samples_per_epoch=16,
batch_size=4,
seed=31,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
default_batches = list(TaskMixtureSampler(**common_kwargs))
assert dataset.calls == 0
assert any(sample_id < 10 or sample_id >= 14 for batch in default_batches for _, sample_id in batch)
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
shard_batches = list(TaskMixtureSampler(**common_kwargs))
assert dataset.calls == len(shard_batches)
assert all([sample_id for _, sample_id in batch] == [10, 11, 12, 13] for batch in shard_batches)
def test_task_mixture_sampler_emits_oss_hints_and_worker_lanes(monkeypatch):
class _FakeOssDataset:
def __init__(self):
self.lanes = []
def sample_shard_aware_indices(self, batch_size, generator, lane_id=0):
self.lanes.append(lane_id)
return OssShardBatch(
indices=list(range(lane_id * 100, lane_id * 100 + batch_size)),
current_shard=f"/oss/current-{lane_id}.safetensors",
next_shard=f"/oss/next-{lane_id}.safetensors",
shard_id=lane_id,
)
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_BLOCK_BATCHES", "4")
monkeypatch.setenv("PSI_TASK_MIXTURE_RANK_LOCAL_BLOCKS", "1")
monkeypatch.setenv("PSI_OMNI_OSS_WORKER_LANES", "2")
dataset = _FakeOssDataset()
sampler = TaskMixtureSampler(
dataset_lens=[1000],
datasets=[dataset],
mixture_ratios=[1],
num_samples_per_epoch=32,
batch_size=4,
seed=37,
)
batches = list(sampler)
assert dataset.lanes == [0, 1, 0, 1, 0, 1, 0, 1]
assert all(len(item) == 4 for batch in batches for item in batch)
assert all(item[2] == f"/oss/current-{batch_id % 2}.safetensors" for batch_id, batch in enumerate(batches) for item in batch)