Download src/psi/data/sampler_test.py from Tinghong-Ye22/code-Psi0-yth-v21: direct link, hf CLI and curl.
- Browser
- Download file 2.25 kB
-
https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/src/psi/data/sampler_test.py
- Command line
-
hf download hf://Tinghong-Ye22/code-Psi0-yth-v21/src/psi/data/sampler_test.py
-
curl -L -o sampler_test.py https://huggingface.co/Tinghong-Ye22/code-Psi0-yth-v21/resolve/main/src/psi/data/sampler_test.py
2.25 kB
| from collections import Counter | |
| from psi.data.sampler import TaskMixtureSampler | |
| def _dataset_sequence(sampler): | |
| sequence = [] | |
| for batch in sampler: | |
| dataset_ids = {dataset_id for dataset_id, _ in batch} | |
| assert len(dataset_ids) == 1 | |
| sequence.append(next(iter(dataset_ids))) | |
| return sequence | |
| def test_task_mixture_sampler_batches_are_homogeneous(monkeypatch): | |
| monkeypatch.setenv("RANK", "0") | |
| monkeypatch.setenv("WORLD_SIZE", "1") | |
| sampler = TaskMixtureSampler( | |
| dataset_lens=[100, 50, 30], | |
| mixture_ratios=[3, 1, 0], | |
| num_samples_per_epoch=64, | |
| batch_size=4, | |
| seed=7, | |
| ) | |
| batches = list(sampler) | |
| assert len(batches) == 16 | |
| for batch in batches: | |
| assert len(batch) == 4 | |
| assert len({dataset_id for dataset_id, _ in batch}) == 1 | |
| def test_task_mixture_sampler_respects_batch_ratios(monkeypatch): | |
| monkeypatch.setenv("RANK", "0") | |
| monkeypatch.setenv("WORLD_SIZE", "1") | |
| sampler = TaskMixtureSampler( | |
| dataset_lens=[100, 50], | |
| mixture_ratios=[3, 1], | |
| num_samples_per_epoch=64, | |
| batch_size=4, | |
| seed=11, | |
| ) | |
| counts = Counter(_dataset_sequence(sampler)) | |
| assert counts == Counter({0: 12, 1: 4}) | |
| def test_task_mixture_sampler_distributed_schedule_matches_global(monkeypatch): | |
| common_kwargs = dict( | |
| dataset_lens=[100, 50, 30], | |
| mixture_ratios=[5, 3, 2], | |
| num_samples_per_epoch=64, | |
| batch_size=4, | |
| seed=19, | |
| ) | |
| monkeypatch.setenv("RANK", "0") | |
| monkeypatch.setenv("WORLD_SIZE", "1") | |
| global_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs)) | |
| monkeypatch.setenv("RANK", "0") | |
| monkeypatch.setenv("WORLD_SIZE", "2") | |
| rank0_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs)) | |
| monkeypatch.setenv("RANK", "1") | |
| monkeypatch.setenv("WORLD_SIZE", "2") | |
| rank1_sequence = _dataset_sequence(TaskMixtureSampler(**common_kwargs)) | |
| merged_sequence = [] | |
| for batch_idx in range(len(global_sequence)): | |
| local_idx = batch_idx // 2 | |
| merged_sequence.append(rank0_sequence[local_idx] if batch_idx % 2 == 0 else rank1_sequence[local_idx]) | |
| assert merged_sequence == global_sequence | |