from collections import Counter from psi.data.sampler import TaskMixtureSampler class _FakeShardAwareDataset: def __init__(self): self.calls = 0 def sample_shard_aware_indices(self, batch_size, generator): self.calls += 1 return list(range(10, 10 + batch_size)) def _dataset_sequence(sampler): sequence = [] for batch in sampler: dataset_ids = {dataset_id for dataset_id, _ in batch} assert len(dataset_ids) == 1 sequence.append(next(iter(dataset_ids))) return sequence def test_task_mixture_sampler_batches_are_homogeneous(monkeypatch): monkeypatch.setenv("RANK", "0") monkeypatch.setenv("WORLD_SIZE", "1") monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0") sampler = TaskMixtureSampler( dataset_lens=[100, 50, 30], mixture_ratios=[3, 1, 0], num_samples_per_epoch=64, batch_size=4, seed=7, ) batches = list(sampler) assert len(batches) == 16 for batch in batches: assert len(batch) == 4 assert len({dataset_id for dataset_id, _ in batch}) == 1 def test_task_mixture_sampler_respects_batch_ratios(monkeypatch): monkeypatch.setenv("RANK", "0") monkeypatch.setenv("WORLD_SIZE", "1") monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0") sampler = TaskMixtureSampler( dataset_lens=[100, 50], mixture_ratios=[3, 1], num_samples_per_epoch=64, batch_size=4, seed=11, ) counts = Counter(_dataset_sequence(sampler)) assert counts == Counter({0: 12, 1: 4}) def test_task_mixture_sampler_distributed_schedule_matches_global(monkeypatch): monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "0") common_kwargs = dict( dataset_lens=[100, 50, 30], mixture_ratios=[5, 3, 2], num_samples_per_epoch=64, batch_size=4, seed=19, ) monkeypatch.setenv("RANK", "0") monkeypatch.setenv("WORLD_SIZE", "1") global_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs)) monkeypatch.setenv("RANK", "0") monkeypatch.setenv("WORLD_SIZE", "2") rank0_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs)) monkeypatch.setenv("RANK", "1") monkeypatch.setenv("WORLD_SIZE", "2") rank1_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs)) merged_sequence = [] for batch_idx in range(len(global_sequence)): local_idx = batch_idx // 2 merged_sequence.append(rank0_sequence[local_idx] if batch_idx % 2 == 0 else rank1_sequence[local_idx]) assert merged_sequence == global_sequence def test_task_mixture_sampler_can_sync_tasks_across_ranks(monkeypatch): common_kwargs = dict( dataset_lens=[100, 50, 30], mixture_ratios=[5, 3, 2], num_samples_per_epoch=64, batch_size=4, seed=23, ) monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "1") monkeypatch.setenv("WORLD_SIZE", "2") monkeypatch.setenv("RANK", "0") rank0_batches = list(TaskMixtureSampler(**common_kwargs)) rank0_sequence = _dataset_sequence(rank0_batches) monkeypatch.setenv("RANK", "1") rank1_batches = list(TaskMixtureSampler(**common_kwargs)) rank1_sequence = _dataset_sequence(rank1_batches) assert rank0_sequence == rank1_sequence assert all( {sample_id for _, sample_id in batch0} != {sample_id for _, sample_id in batch1} for batch0, batch1 in zip(rank0_batches, rank1_batches) ) def test_task_mixture_sampler_uses_shard_aware_indices_only_when_enabled(monkeypatch): monkeypatch.setenv("RANK", "0") monkeypatch.setenv("WORLD_SIZE", "1") dataset = _FakeShardAwareDataset() common_kwargs = dict( dataset_lens=[100], datasets=[dataset], mixture_ratios=[1], num_samples_per_epoch=16, batch_size=4, seed=31, ) monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0") default_batches = list(TaskMixtureSampler(**common_kwargs)) assert dataset.calls == 0 assert any(sample_id < 10 or sample_id >= 14 for batch in default_batches for _, sample_id in batch) monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1") shard_batches = list(TaskMixtureSampler(**common_kwargs)) assert dataset.calls == len(shard_batches) assert all([sample_id for _, sample_id in batch] == [10, 11, 12, 13] for batch in shard_batches)