Download src/psi/data/sampler_test.py from Tinghong-Ye22/code-Movens-Zero_train: direct link, hf CLI and curl.
- Browser
- Download file 12.4 kB
-
https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/src/psi/data/sampler_test.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Movens-Zero_train/src/psi/data/sampler_test.py
-
curl -L -o sampler_test.py https://huggingface.co/Tinghong-Ye22/code-Movens-Zero_train/resolve/main/src/psi/data/sampler_test.py
12.4 kB
| from collections import Counter | |
| import torch | |
| from psi.data.sampler import TaskMixtureSampler | |
| from psi.data.seed.omni_oss_sampler import OssShardBatch | |
| class _FakeShardAwareDataset: | |
| def __init__(self): | |
| self.calls = 0 | |
| def sample_shard_aware_indices(self, batch_size, generator): | |
| self.calls += 1 | |
| return list(range(10, 10 + batch_size)) | |
| def _dataset_sequence(sampler): | |
| sequence = [] | |
| for batch in sampler: | |
| dataset_ids = {item[0] for item in batch} | |
| assert len(dataset_ids) == 1 | |
| sequence.append(next(iter(dataset_ids))) | |
| return sequence | |
| def test_task_mixture_sampler_batches_are_homogeneous(monkeypatch): | |
| monkeypatch.setenv("RANK", "0") | |
| monkeypatch.setenv("WORLD_SIZE", "1") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0") | |
| sampler = TaskMixtureSampler( | |
| dataset_lens=[100, 50, 30], | |
| mixture_ratios=[3, 1, 0], | |
| num_samples_per_epoch=64, | |
| batch_size=4, | |
| seed=7, | |
| ) | |
| batches = list(sampler) | |
| assert len(batches) == 16 | |
| for batch in batches: | |
| assert len(batch) == 4 | |
| assert len({dataset_id for dataset_id, _ in batch}) == 1 | |
| def test_task_mixture_sampler_respects_batch_ratios(monkeypatch): | |
| monkeypatch.setenv("RANK", "0") | |
| monkeypatch.setenv("WORLD_SIZE", "1") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0") | |
| sampler = TaskMixtureSampler( | |
| dataset_lens=[100, 50], | |
| mixture_ratios=[3, 1], | |
| num_samples_per_epoch=64, | |
| batch_size=4, | |
| seed=11, | |
| ) | |
| counts = Counter(_dataset_sequence(sampler)) | |
| assert counts == Counter({0: 12, 1: 4}) | |
| def test_task_mixture_sampler_distributed_schedule_matches_global(monkeypatch): | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "0") | |
| common_kwargs = dict( | |
| dataset_lens=[100, 50, 30], | |
| mixture_ratios=[5, 3, 2], | |
| num_samples_per_epoch=64, | |
| batch_size=4, | |
| seed=19, | |
| ) | |
| monkeypatch.setenv("RANK", "0") | |
| monkeypatch.setenv("WORLD_SIZE", "1") | |
| global_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs)) | |
| monkeypatch.setenv("RANK", "0") | |
| monkeypatch.setenv("WORLD_SIZE", "2") | |
| rank0_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs)) | |
| monkeypatch.setenv("RANK", "1") | |
| monkeypatch.setenv("WORLD_SIZE", "2") | |
| rank1_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs)) | |
| merged_sequence = [] | |
| for batch_idx in range(len(global_sequence)): | |
| local_idx = batch_idx // 2 | |
| merged_sequence.append(rank0_sequence[local_idx] if batch_idx % 2 == 0 else rank1_sequence[local_idx]) | |
| assert merged_sequence == global_sequence | |
| def test_task_mixture_sampler_can_sync_tasks_across_ranks(monkeypatch): | |
| common_kwargs = dict( | |
| dataset_lens=[100, 50, 30], | |
| mixture_ratios=[5, 3, 2], | |
| num_samples_per_epoch=64, | |
| batch_size=4, | |
| seed=23, | |
| ) | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_SYNC_RANKS", "1") | |
| monkeypatch.setenv("WORLD_SIZE", "2") | |
| monkeypatch.setenv("RANK", "0") | |
| rank0_batches = list(TaskMixtureSampler(**common_kwargs)) | |
| rank0_sequence = _dataset_sequence(rank0_batches) | |
| monkeypatch.setenv("RANK", "1") | |
| rank1_batches = list(TaskMixtureSampler(**common_kwargs)) | |
| rank1_sequence = _dataset_sequence(rank1_batches) | |
| assert rank0_sequence == rank1_sequence | |
| assert all( | |
| {sample_id for _, sample_id in batch0} != {sample_id for _, sample_id in batch1} | |
| for batch0, batch1 in zip(rank0_batches, rank1_batches) | |
| ) | |
| def test_task_mixture_sampler_balances_equal_tasks_across_ranks(monkeypatch): | |
| common_kwargs = dict( | |
| dataset_lens=[100, 100, 100, 100], | |
| mixture_ratios=[1, 1, 1, 1], | |
| num_samples_per_epoch=64, | |
| batch_size=4, | |
| seed=23, | |
| ) | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0") | |
| monkeypatch.setenv("WORLD_SIZE", "8") | |
| rank_sequences = [] | |
| for rank in range(8): | |
| monkeypatch.setenv("RANK", str(rank)) | |
| batches = list(TaskMixtureSampler(**common_kwargs)) | |
| assert batches | |
| task_ids = [{item[0] for item in batch} for batch in batches] | |
| assert all(len(ids) == 1 for ids in task_ids) | |
| rank_sequences.append([next(iter(ids)) for ids in task_ids]) | |
| # Every local step has exactly two ranks assigned to each enabled task. | |
| for local_step in range(len(rank_sequences[0])): | |
| assert Counter(sequence[local_step] for sequence in rank_sequences) == Counter( | |
| {0: 2, 1: 2, 2: 2, 3: 2} | |
| ) | |
| def test_task_mixture_sampler_assigns_exact_weighted_rank_counts(monkeypatch): | |
| common_kwargs = dict( | |
| dataset_lens=[100, 100, 100, 100], | |
| mixture_ratios=[3, 1, 3, 1], | |
| num_samples_per_epoch=64, | |
| batch_size=4, | |
| seed=29, | |
| ) | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0") | |
| monkeypatch.setenv("WORLD_SIZE", "8") | |
| rank_sequences = [] | |
| for rank in range(8): | |
| monkeypatch.setenv("RANK", str(rank)) | |
| batches = list(TaskMixtureSampler(**common_kwargs)) | |
| assert batches | |
| rank_sequences.append(_dataset_sequence(batches)) | |
| for local_step in range(len(rank_sequences[0])): | |
| assert Counter(sequence[local_step] for sequence in rank_sequences) == Counter( | |
| {0: 3, 1: 1, 2: 3, 3: 1} | |
| ) | |
| def test_task_mixture_sampler_uses_only_residual_flex_ranks_for_fractional_weights(monkeypatch): | |
| common_kwargs = dict( | |
| dataset_lens=[19, 6, 19, 56], | |
| mixture_ratios=[19, 6, 19, 56], | |
| num_samples_per_epoch=8000, | |
| batch_size=10, | |
| seed=31, | |
| ) | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_BLOCK_BATCHES", "8") | |
| monkeypatch.setenv("WORLD_SIZE", "8") | |
| rank_sequences = [] | |
| fixed_ranks = 0 | |
| for rank in range(8): | |
| monkeypatch.setenv("RANK", str(rank)) | |
| sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs)) | |
| rank_sequences.append(sequence) | |
| fixed_ranks += int(len(set(sequence)) == 1) | |
| # floor([1.52, 0.48, 1.52, 4.48]) keeps six core ranks resident; | |
| # only two flex ranks realize the residual proportions. | |
| assert fixed_ranks == 6 | |
| assert Counter(task for sequence in rank_sequences for task in sequence) == Counter( | |
| {0: 152, 1: 48, 2: 152, 3: 448} | |
| ) | |
| def test_fractional_flex_rank_initializes_all_tasks_before_sampling(monkeypatch): | |
| class _InitAwareDataset: | |
| def __init__(self): | |
| self.batch_sizes = [] | |
| def sample_shard_aware_indices(self, batch_size, generator, lane_id=0): | |
| del generator, lane_id | |
| self.batch_sizes.append(batch_size) | |
| return list(range(batch_size)) | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1") | |
| monkeypatch.setenv("WORLD_SIZE", "8") | |
| datasets = [_InitAwareDataset() for _ in range(4)] | |
| common_kwargs = dict( | |
| dataset_lens=[19, 6, 19, 56], | |
| datasets=datasets, | |
| mixture_ratios=[19, 6, 19, 56], | |
| num_samples_per_epoch=800, | |
| batch_size=10, | |
| seed=41, | |
| ) | |
| flex_rank = None | |
| for rank in range(8): | |
| monkeypatch.setenv("RANK", str(rank)) | |
| plan = TaskMixtureSampler(**common_kwargs)._balanced_rank_plan() | |
| if plan is not None and plan[1] is not None: | |
| flex_rank = rank | |
| break | |
| assert flex_rank is not None | |
| monkeypatch.setenv("RANK", str(flex_rank)) | |
| list(TaskMixtureSampler(**common_kwargs)) | |
| assert all(dataset.batch_sizes[0] == 0 for dataset in datasets) | |
| assert all(any(batch_size > 0 for batch_size in dataset.batch_sizes) for dataset in datasets) | |
| def test_task_mixture_sampler_uses_shard_aware_indices_only_when_enabled(monkeypatch): | |
| monkeypatch.setenv("RANK", "0") | |
| monkeypatch.setenv("WORLD_SIZE", "1") | |
| dataset = _FakeShardAwareDataset() | |
| common_kwargs = dict( | |
| dataset_lens=[100], | |
| datasets=[dataset], | |
| mixture_ratios=[1], | |
| num_samples_per_epoch=16, | |
| batch_size=4, | |
| seed=31, | |
| ) | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "0") | |
| default_batches = list(TaskMixtureSampler(**common_kwargs)) | |
| assert dataset.calls == 0 | |
| assert any(sample_id < 10 or sample_id >= 14 for batch in default_batches for _, sample_id in batch) | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1") | |
| shard_batches = list(TaskMixtureSampler(**common_kwargs)) | |
| assert dataset.calls == len(shard_batches) | |
| assert all([sample_id for _, sample_id in batch] == [10, 11, 12, 13] for batch in shard_batches) | |
| def test_task_mixture_sampler_emits_oss_hints_and_worker_lanes(monkeypatch): | |
| class _FakeOssDataset: | |
| def __init__(self): | |
| self.lanes = [] | |
| def sample_shard_aware_indices(self, batch_size, generator, lane_id=0): | |
| self.lanes.append(lane_id) | |
| return OssShardBatch( | |
| indices=list(range(lane_id * 100, lane_id * 100 + batch_size)), | |
| current_shard=f"/oss/current-{lane_id}.safetensors", | |
| next_shard=f"/oss/next-{lane_id}.safetensors", | |
| shard_id=lane_id, | |
| ) | |
| monkeypatch.setenv("RANK", "0") | |
| monkeypatch.setenv("WORLD_SIZE", "1") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_BLOCK_BATCHES", "4") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_RANK_LOCAL_BLOCKS", "1") | |
| monkeypatch.setenv("PSI_OMNI_OSS_WORKER_LANES", "2") | |
| dataset = _FakeOssDataset() | |
| sampler = TaskMixtureSampler( | |
| dataset_lens=[1000], | |
| datasets=[dataset], | |
| mixture_ratios=[1], | |
| num_samples_per_epoch=32, | |
| batch_size=4, | |
| seed=37, | |
| ) | |
| batches = list(sampler) | |
| assert dataset.lanes == [0, 1, 0, 1, 0, 1, 0, 1] | |
| assert all(len(item) == 4 for batch in batches for item in batch) | |
| assert all(item[2] == f"/oss/current-{batch_id % 2}.safetensors" for batch_id, batch in enumerate(batches) for item in batch) | |
| def test_task_mixture_sampler_fast_forward_matches_exact_continuation(monkeypatch): | |
| class _StatefulShardDataset: | |
| def __init__(self, dataset_id): | |
| self.dataset_id = dataset_id | |
| self.lane_calls = Counter() | |
| self.getitem_calls = 0 | |
| def sample_shard_aware_indices(self, batch_size, generator, lane_id=0): | |
| call = self.lane_calls[lane_id] | |
| self.lane_calls[lane_id] += 1 | |
| random_part = torch.randint(0, 1000, (batch_size,), generator=generator).tolist() | |
| return [ | |
| self.dataset_id * 100_000 + lane_id * 10_000 + call * 1000 + value | |
| for value in random_part | |
| ] | |
| def __getitem__(self, index): | |
| self.getitem_calls += 1 | |
| return index | |
| monkeypatch.setenv("WORLD_SIZE", "8") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_BALANCED_RANKS", "1") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_SHARD_AWARE", "1") | |
| monkeypatch.setenv("PSI_TASK_MIXTURE_BLOCK_BATCHES", "8") | |
| monkeypatch.setenv("PSI_OMNI_OSS_WORKER_LANES", "3") | |
| common_kwargs = dict( | |
| dataset_lens=[190_000, 60_000, 190_000, 560_000], | |
| mixture_ratios=[19, 6, 19, 56], | |
| num_samples_per_epoch=8 * 40 * 4, | |
| batch_size=4, | |
| seed=43, | |
| ) | |
| offset = 17 | |
| # Cover both fixed and residual-flex ranks from the production schedule. | |
| for rank in range(8): | |
| monkeypatch.setenv("RANK", str(rank)) | |
| baseline_datasets = [_StatefulShardDataset(i) for i in range(4)] | |
| baseline = TaskMixtureSampler(datasets=baseline_datasets, **common_kwargs) | |
| baseline.set_epoch(1) | |
| expected = list(baseline)[offset:] | |
| resumed_datasets = [_StatefulShardDataset(i) for i in range(4)] | |
| resumed = TaskMixtureSampler(datasets=resumed_datasets, **common_kwargs) | |
| resumed.set_epoch(1) | |
| reports = [] | |
| assert resumed.fast_forward(offset, progress_callback=reports.append) == offset | |
| assert reports[-1] == offset | |
| assert list(resumed) == expected | |
| assert sum(dataset.getitem_calls for dataset in resumed_datasets) == 0 | |