Download src/psi/trainers/distributed_flow_sampling_test.py from Tinghong-Ye22/code-HFD: direct link, hf CLI and curl.
- Browser
- Download file 4.99 kB
-
https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/trainers/distributed_flow_sampling_test.py
- Command line
-
hf download hf://Tinghong-Ye22/code-HFD/src/psi/trainers/distributed_flow_sampling_test.py
-
curl -L -o distributed_flow_sampling_test.py https://huggingface.co/Tinghong-Ye22/code-HFD/resolve/main/src/psi/trainers/distributed_flow_sampling_test.py
4.99 kB
| from collections import Counter | |
| import pytest | |
| import torch | |
| from psi.trainers.distributed_flow_sampling import ( | |
| DelaySamplingConfig, | |
| DistributedFlowRandomSampler, | |
| ) | |
| def _samplers( | |
| *, | |
| world_size: int, | |
| seed: int = 292285, | |
| delay: DelaySamplingConfig | None = None, | |
| ): | |
| return [ | |
| DistributedFlowRandomSampler( | |
| seed=seed, | |
| rank=rank, | |
| world_size=world_size, | |
| sigma_min=0.1, | |
| sigma_max=1.0, | |
| delay=delay, | |
| ) | |
| for rank in range(world_size) | |
| ] | |
| def test_sigma_has_one_jittered_sample_per_global_stratum(): | |
| world_size = 4 | |
| local_batch_size = 8 | |
| outputs = [ | |
| sampler.sample((local_batch_size, 3, 2), device="cpu").sigmas | |
| for sampler in _samplers(world_size=world_size) | |
| ] | |
| global_sigmas = torch.cat(outputs) | |
| global_batch_size = world_size * local_batch_size | |
| unit_sigmas = (global_sigmas.double() - 0.1) / 0.9 | |
| occupied_strata = torch.floor(unit_sigmas * global_batch_size).long().sort().values | |
| assert torch.equal(occupied_strata, torch.arange(global_batch_size)) | |
| assert torch.all(global_sigmas >= 0.1) | |
| assert torch.all(global_sigmas < 1.0) | |
| def test_rank_noise_is_independent_but_exactly_reproducible(): | |
| rank0, rank1 = _samplers(world_size=2) | |
| noise0 = rank0.sample((16, 4, 3), device="cpu").noise | |
| noise1 = rank1.sample((16, 4, 3), device="cpu").noise | |
| replay0 = _samplers(world_size=2)[0].sample((16, 4, 3), device="cpu").noise | |
| assert not torch.equal(noise0, noise1) | |
| assert torch.equal(noise0, replay0) | |
| def test_fixed_delay_matches_deployment_value_on_every_rank(): | |
| samplers = _samplers( | |
| world_size=4, | |
| delay=DelaySamplingConfig.fixed(deployment_delay=9), | |
| ) | |
| delays = torch.cat( | |
| [sampler.sample((7, 2), device="cpu").delay for sampler in samplers] | |
| ) | |
| assert torch.equal(delays, torch.full((28,), 9, dtype=torch.long)) | |
| def test_explicit_delay_distribution_is_balanced_over_global_batch(): | |
| delay = DelaySamplingConfig.balanced( | |
| values=(0, 5, 10), | |
| probabilities=(0.25, 0.5, 0.25), | |
| ) | |
| samplers = _samplers(world_size=4, delay=delay) | |
| delays = torch.cat( | |
| [sampler.sample((8, 2), device="cpu").delay for sampler in samplers] | |
| ) | |
| assert Counter(delays.tolist()) == Counter({0: 8, 5: 16, 10: 8}) | |
| # The deterministic permutation should not leave categories as contiguous | |
| # blocks even though systematic sampling creates them in CDF order first. | |
| assert len(torch.unique(delays[:8])) > 1 | |
| def test_checkpoint_state_reproduces_the_exact_next_microbatch(): | |
| delay = DelaySamplingConfig.uniform(max_delay=11) | |
| sampler = DistributedFlowRandomSampler( | |
| seed=17, | |
| rank=2, | |
| world_size=4, | |
| delay=delay, | |
| ) | |
| sampler.sample((6, 5, 3), device="cpu") | |
| checkpoint = sampler.state_dict() | |
| expected = sampler.sample((6, 5, 3), device="cpu") | |
| resumed = DistributedFlowRandomSampler( | |
| seed=17, | |
| rank=2, | |
| world_size=4, | |
| delay=delay, | |
| ) | |
| resumed.load_state_dict(checkpoint) | |
| actual = resumed.sample((6, 5, 3), device="cpu") | |
| assert actual.draw_index == expected.draw_index == 1 | |
| assert torch.equal(actual.sigmas, expected.sigmas) | |
| assert torch.equal(actual.delay, expected.delay) | |
| assert torch.equal(actual.noise, expected.noise) | |
| assert resumed.draw_index == sampler.draw_index == 2 | |
| def test_component_streams_do_not_shift_when_delay_policy_changes(): | |
| without_rtc = DistributedFlowRandomSampler(seed=31, rank=1, world_size=2) | |
| with_rtc = DistributedFlowRandomSampler( | |
| seed=31, | |
| rank=1, | |
| world_size=2, | |
| delay=DelaySamplingConfig.uniform(max_delay=11), | |
| ) | |
| plain = without_rtc.sample((9, 4, 3), device="cpu") | |
| rtc = with_rtc.sample((9, 4, 3), device="cpu") | |
| assert torch.equal(plain.sigmas, rtc.sigmas) | |
| assert torch.equal(plain.noise, rtc.noise) | |
| def test_resume_rejects_silent_sampling_configuration_changes(): | |
| # The production default is now sigma_min=0.05. Keep the old value | |
| # explicit here so this test still exercises a real resume mismatch. | |
| original = DistributedFlowRandomSampler( | |
| seed=11, rank=0, world_size=2, sigma_min=0.1 | |
| ) | |
| changed = DistributedFlowRandomSampler( | |
| seed=11, | |
| rank=0, | |
| world_size=2, | |
| sigma_min=0.05, | |
| ) | |
| with pytest.raises(ValueError, match="configuration changed"): | |
| changed.load_state_dict(original.state_dict()) | |
| def test_invalid_delay_config_is_rejected(config): | |
| with pytest.raises(ValueError): | |
| config() | |