code-Movens-Zero_train / src /psi /data /sampler_test.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
980282b verified
Raw History Blame Contribute Delete
12.4 kB
from collections import Counter
import torch
from psi.data.sampler import TaskMixtureSampler
from psi.data.seed.omni_oss_sampler import OssShardBatch
class _FakeShardAwareDataset:
def __init__(self):
self.calls = 0
def sample_shard_aware_indices(self, batch_size, generator):
self.calls += 1
return list(range(10, 10 + batch_size))
def _dataset_sequence(sampler):
sequence = []
for batch in sampler:
dataset_ids = {item[0] for item in batch}
assert len(dataset_ids) == 1
sequence.append(next(iter(dataset_ids)))
return sequence
def test_task_mixture_sampler_batches_are_homogeneous(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
sampler = TaskMixtureSampler(
dataset_lens=[100, 50, 30],
mixture_ratios=[3, 1, 0],
num_samples_per_epoch=64,
batch_size=4,
seed=7,
)
batches = list(sampler)
assert len(batches) == 16
for batch in batches:
assert len(batch) == 4
assert len({dataset_id for dataset_id, _ in batch}) == 1
def test_task_mixture_sampler_respects_batch_ratios(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
sampler = TaskMixtureSampler(
dataset_lens=[100, 50],
mixture_ratios=[3, 1],
num_samples_per_epoch=64,
batch_size=4,
seed=11,
)
counts = Counter(_dataset_sequence(sampler))
assert counts == Counter({0: 12, 1: 4})
def test_task_mixture_sampler_distributed_schedule_matches_global(monkeypatch):
monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "0")
common_kwargs = dict(
dataset_lens=[100, 50, 30],
mixture_ratios=[5, 3, 2],
num_samples_per_epoch=64,
batch_size=4,
seed=19,
)
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
global_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "2")
rank0_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
monkeypatch.setenv("RANK", "1")
monkeypatch.setenv("WORLD_SIZE", "2")
rank1_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
merged_sequence = []
for batch_idx in range(len(global_sequence)):
local_idx = batch_idx // 2
merged_sequence.append(rank0_sequence[local_idx] if batch_idx % 2 == 0 else rank1_sequence[local_idx])
assert merged_sequence == global_sequence
def test_task_mixture_sampler_can_sync_tasks_across_ranks(monkeypatch):
common_kwargs = dict(
dataset_lens=[100, 50, 30],
mixture_ratios=[5, 3, 2],
num_samples_per_epoch=64,
batch_size=4,
seed=23,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "1")
monkeypatch.setenv("WORLD_SIZE", "2")
monkeypatch.setenv("RANK", "0")
rank0_batches = list(TaskMixtureSampler(**common_kwargs))
rank0_sequence = _dataset_sequence(rank0_batches)
monkeypatch.setenv("RANK", "1")
rank1_batches = list(TaskMixtureSampler(**common_kwargs))
rank1_sequence = _dataset_sequence(rank1_batches)
assert rank0_sequence == rank1_sequence
assert all(
{sample_id for _, sample_id in batch0} != {sample_id for _, sample_id in batch1}
for batch0, batch1 in zip(rank0_batches, rank1_batches)
)
def test_task_mixture_sampler_balances_equal_tasks_across_ranks(monkeypatch):
common_kwargs = dict(
dataset_lens=[100, 100, 100, 100],
mixture_ratios=[1, 1, 1, 1],
num_samples_per_epoch=64,
batch_size=4,
seed=23,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
monkeypatch.setenv("WORLD_SIZE", "8")
rank_sequences = []
for rank in range(8):
monkeypatch.setenv("RANK", str(rank))
batches = list(TaskMixtureSampler(**common_kwargs))
assert batches
task_ids = [{item[0] for item in batch} for batch in batches]
assert all(len(ids) == 1 for ids in task_ids)
rank_sequences.append([next(iter(ids)) for ids in task_ids])
# Every local step has exactly two ranks assigned to each enabled task.
for local_step in range(len(rank_sequences[0])):
assert Counter(sequence[local_step] for sequence in rank_sequences) == Counter(
{0: 2, 1: 2, 2: 2, 3: 2}
)
def test_task_mixture_sampler_assigns_exact_weighted_rank_counts(monkeypatch):
common_kwargs = dict(
dataset_lens=[100, 100, 100, 100],
mixture_ratios=[3, 1, 3, 1],
num_samples_per_epoch=64,
batch_size=4,
seed=29,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
monkeypatch.setenv("WORLD_SIZE", "8")
rank_sequences = []
for rank in range(8):
monkeypatch.setenv("RANK", str(rank))
batches = list(TaskMixtureSampler(**common_kwargs))
assert batches
rank_sequences.append(_dataset_sequence(batches))
for local_step in range(len(rank_sequences[0])):
assert Counter(sequence[local_step] for sequence in rank_sequences) == Counter(
{0: 3, 1: 1, 2: 3, 3: 1}
)
def test_task_mixture_sampler_uses_only_residual_flex_ranks_for_fractional_weights(monkeypatch):
common_kwargs = dict(
dataset_lens=[19, 6, 19, 56],
mixture_ratios=[19, 6, 19, 56],
num_samples_per_epoch=8000,
batch_size=10,
seed=31,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
monkeypatch.setenv("PSI_TASK_MIXTURE_BLOCK_BATCHES", "8")
monkeypatch.setenv("WORLD_SIZE", "8")
rank_sequences = []
fixed_ranks = 0
for rank in range(8):
monkeypatch.setenv("RANK", str(rank))
sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs))
rank_sequences.append(sequence)
fixed_ranks += int(len(set(sequence)) == 1)
# floor([1.52, 0.48, 1.52, 4.48]) keeps six core ranks resident;
# only two flex ranks realize the residual proportions.
assert fixed_ranks == 6
assert Counter(task for sequence in rank_sequences for task in sequence) == Counter(
{0: 152, 1: 48, 2: 152, 3: 448}
)
def test_fractional_flex_rank_initializes_all_tasks_before_sampling(monkeypatch):
class _InitAwareDataset:
def __init__(self):
self.batch_sizes = []
def sample_shard_aware_indices(self, batch_size, generator, lane_id=0):
del generator, lane_id
self.batch_sizes.append(batch_size)
return list(range(batch_size))
monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
monkeypatch.setenv("WORLD_SIZE", "8")
datasets = [_InitAwareDataset() for _ in range(4)]
common_kwargs = dict(
dataset_lens=[19, 6, 19, 56],
datasets=datasets,
mixture_ratios=[19, 6, 19, 56],
num_samples_per_epoch=800,
batch_size=10,
seed=41,
)
flex_rank = None
for rank in range(8):
monkeypatch.setenv("RANK", str(rank))
plan = TaskMixtureSampler(**common_kwargs)._balanced_rank_plan()
if plan is not None and plan[1] is not None:
flex_rank = rank
break
assert flex_rank is not None
monkeypatch.setenv("RANK", str(flex_rank))
list(TaskMixtureSampler(**common_kwargs))
assert all(dataset.batch_sizes[0] == 0 for dataset in datasets)
assert all(any(batch_size > 0 for batch_size in dataset.batch_sizes) for dataset in datasets)
def test_task_mixture_sampler_uses_shard_aware_indices_only_when_enabled(monkeypatch):
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
dataset = _FakeShardAwareDataset()
common_kwargs = dict(
dataset_lens=[100],
datasets=[dataset],
mixture_ratios=[1],
num_samples_per_epoch=16,
batch_size=4,
seed=31,
)
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0")
default_batches = list(TaskMixtureSampler(**common_kwargs))
assert dataset.calls == 0
assert any(sample_id < 10 or sample_id >= 14 for batch in default_batches for _, sample_id in batch)
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
shard_batches = list(TaskMixtureSampler(**common_kwargs))
assert dataset.calls == len(shard_batches)
assert all([sample_id for _, sample_id in batch] == [10, 11, 12, 13] for batch in shard_batches)
def test_task_mixture_sampler_emits_oss_hints_and_worker_lanes(monkeypatch):
class _FakeOssDataset:
def __init__(self):
self.lanes = []
def sample_shard_aware_indices(self, batch_size, generator, lane_id=0):
self.lanes.append(lane_id)
return OssShardBatch(
indices=list(range(lane_id * 100, lane_id * 100 + batch_size)),
current_shard=f"/oss/current-{lane_id}.safetensors",
next_shard=f"/oss/next-{lane_id}.safetensors",
shard_id=lane_id,
)
monkeypatch.setenv("RANK", "0")
monkeypatch.setenv("WORLD_SIZE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_BLOCK_BATCHES", "4")
monkeypatch.setenv("PSI_TASK_MIXTURE_RANK_LOCAL_BLOCKS", "1")
monkeypatch.setenv("PSI_OMNI_OSS_WORKER_LANES", "2")
dataset = _FakeOssDataset()
sampler = TaskMixtureSampler(
dataset_lens=[1000],
datasets=[dataset],
mixture_ratios=[1],
num_samples_per_epoch=32,
batch_size=4,
seed=37,
)
batches = list(sampler)
assert dataset.lanes == [0, 1, 0, 1, 0, 1, 0, 1]
assert all(len(item) == 4 for batch in batches for item in batch)
assert all(item[2] == f"/oss/current-{batch_id % 2}.safetensors" for batch_id, batch in enumerate(batches) for item in batch)
def test_task_mixture_sampler_fast_forward_matches_exact_continuation(monkeypatch):
class _StatefulShardDataset:
def __init__(self, dataset_id):
self.dataset_id = dataset_id
self.lane_calls = Counter()
self.getitem_calls = 0
def sample_shard_aware_indices(self, batch_size, generator, lane_id=0):
call = self.lane_calls[lane_id]
self.lane_calls[lane_id] += 1
random_part = torch.randint(0, 1000, (batch_size,), generator=generator).tolist()
return [
self.dataset_id * 100_000 + lane_id * 10_000 + call * 1000 + value
for value in random_part
]
def __getitem__(self, index):
self.getitem_calls += 1
return index
monkeypatch.setenv("WORLD_SIZE", "8")
monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1")
monkeypatch.setenv("PSI_TASK_MIXTURE_BLOCK_BATCHES", "8")
monkeypatch.setenv("PSI_OMNI_OSS_WORKER_LANES", "3")
common_kwargs = dict(
dataset_lens=[190_000, 60_000, 190_000, 560_000],
mixture_ratios=[19, 6, 19, 56],
num_samples_per_epoch=8 * 40 * 4,
batch_size=4,
seed=43,
)
offset = 17
# Cover both fixed and residual-flex ranks from the production schedule.
for rank in range(8):
monkeypatch.setenv("RANK", str(rank))
baseline_datasets = [_StatefulShardDataset(i) for i in range(4)]
baseline = TaskMixtureSampler(datasets=baseline_datasets, **common_kwargs)
baseline.set_epoch(1)
expected = list(baseline)[offset:]
resumed_datasets = [_StatefulShardDataset(i) for i in range(4)]
resumed = TaskMixtureSampler(datasets=resumed_datasets, **common_kwargs)
resumed.set_epoch(1)
reports = []
assert resumed.fast_forward(offset, progress_callback=reports.append) == offset
assert reports[-1] == offset
assert list(resumed) == expected
assert sum(dataset.getitem_calls for dataset in resumed_datasets) == 0