from collections import Counter import pytest import torch from psi.trainers.distributed_flow_sampling import ( DelaySamplingConfig, DistributedFlowRandomSampler, ) def _samplers( *, world_size: int, seed: int = 292285, delay: DelaySamplingConfig | None = None, ): return [ DistributedFlowRandomSampler( seed=seed, rank=rank, world_size=world_size, sigma_min=0.1, sigma_max=1.0, delay=delay, ) for rank in range(world_size) ] def test_sigma_has_one_jittered_sample_per_global_stratum(): world_size = 4 local_batch_size = 8 outputs = [ sampler.sample((local_batch_size, 3, 2), device="cpu").sigmas for sampler in _samplers(world_size=world_size) ] global_sigmas = torch.cat(outputs) global_batch_size = world_size * local_batch_size unit_sigmas = (global_sigmas.double() - 0.1) / 0.9 occupied_strata = torch.floor(unit_sigmas * global_batch_size).long().sort().values assert torch.equal(occupied_strata, torch.arange(global_batch_size)) assert torch.all(global_sigmas >= 0.1) assert torch.all(global_sigmas < 1.0) def test_rank_noise_is_independent_but_exactly_reproducible(): rank0, rank1 = _samplers(world_size=2) noise0 = rank0.sample((16, 4, 3), device="cpu").noise noise1 = rank1.sample((16, 4, 3), device="cpu").noise replay0 = _samplers(world_size=2)[0].sample((16, 4, 3), device="cpu").noise assert not torch.equal(noise0, noise1) assert torch.equal(noise0, replay0) def test_fixed_delay_matches_deployment_value_on_every_rank(): samplers = _samplers( world_size=4, delay=DelaySamplingConfig.fixed(deployment_delay=9), ) delays = torch.cat( [sampler.sample((7, 2), device="cpu").delay for sampler in samplers] ) assert torch.equal(delays, torch.full((28,), 9, dtype=torch.long)) def test_explicit_delay_distribution_is_balanced_over_global_batch(): delay = DelaySamplingConfig.balanced( values=(0, 5, 10), probabilities=(0.25, 0.5, 0.25), ) samplers = _samplers(world_size=4, delay=delay) delays = torch.cat( [sampler.sample((8, 2), device="cpu").delay for sampler in samplers] ) assert Counter(delays.tolist()) == Counter({0: 8, 5: 16, 10: 8}) # The deterministic permutation should not leave categories as contiguous # blocks even though systematic sampling creates them in CDF order first. assert len(torch.unique(delays[:8])) > 1 def test_checkpoint_state_reproduces_the_exact_next_microbatch(): delay = DelaySamplingConfig.uniform(max_delay=11) sampler = DistributedFlowRandomSampler( seed=17, rank=2, world_size=4, delay=delay, ) sampler.sample((6, 5, 3), device="cpu") checkpoint = sampler.state_dict() expected = sampler.sample((6, 5, 3), device="cpu") resumed = DistributedFlowRandomSampler( seed=17, rank=2, world_size=4, delay=delay, ) resumed.load_state_dict(checkpoint) actual = resumed.sample((6, 5, 3), device="cpu") assert actual.draw_index == expected.draw_index == 1 assert torch.equal(actual.sigmas, expected.sigmas) assert torch.equal(actual.delay, expected.delay) assert torch.equal(actual.noise, expected.noise) assert resumed.draw_index == sampler.draw_index == 2 def test_component_streams_do_not_shift_when_delay_policy_changes(): without_rtc = DistributedFlowRandomSampler(seed=31, rank=1, world_size=2) with_rtc = DistributedFlowRandomSampler( seed=31, rank=1, world_size=2, delay=DelaySamplingConfig.uniform(max_delay=11), ) plain = without_rtc.sample((9, 4, 3), device="cpu") rtc = with_rtc.sample((9, 4, 3), device="cpu") assert torch.equal(plain.sigmas, rtc.sigmas) assert torch.equal(plain.noise, rtc.noise) def test_resume_rejects_silent_sampling_configuration_changes(): # The production default is now sigma_min=0.05. Keep the old value # explicit here so this test still exercises a real resume mismatch. original = DistributedFlowRandomSampler( seed=11, rank=0, world_size=2, sigma_min=0.1 ) changed = DistributedFlowRandomSampler( seed=11, rank=0, world_size=2, sigma_min=0.05, ) with pytest.raises(ValueError, match="configuration changed"): changed.load_state_dict(original.state_dict()) @pytest.mark.parametrize( "config", [ lambda: DelaySamplingConfig.fixed(-1), lambda: DelaySamplingConfig.balanced((), ()), lambda: DelaySamplingConfig.balanced((0, 1), (1.0,)), lambda: DelaySamplingConfig.balanced((0, 0), (0.5, 0.5)), lambda: DelaySamplingConfig.balanced((0, 1), (0.0, 0.0)), ], ) def test_invalid_delay_config_is_rejected(config): with pytest.raises(ValueError): config()