code-HFD / src /psi /trainers /distributed_flow_sampling_test.py
Tinghong-Ye22's picture
Add files using upload-large-folder tool
06f9561 verified
Raw History Blame Contribute Delete
4.99 kB
from collections import Counter
import pytest
import torch
from psi.trainers.distributed_flow_sampling import (
DelaySamplingConfig,
DistributedFlowRandomSampler,
)
def _samplers(
*,
world_size: int,
seed: int = 292285,
delay: DelaySamplingConfig | None = None,
):
return [
DistributedFlowRandomSampler(
seed=seed,
rank=rank,
world_size=world_size,
sigma_min=0.1,
sigma_max=1.0,
delay=delay,
)
for rank in range(world_size)
]
def test_sigma_has_one_jittered_sample_per_global_stratum():
world_size = 4
local_batch_size = 8
outputs = [
sampler.sample((local_batch_size, 3, 2), device="cpu").sigmas
for sampler in _samplers(world_size=world_size)
]
global_sigmas = torch.cat(outputs)
global_batch_size = world_size * local_batch_size
unit_sigmas = (global_sigmas.double() - 0.1) / 0.9
occupied_strata = torch.floor(unit_sigmas * global_batch_size).long().sort().values
assert torch.equal(occupied_strata, torch.arange(global_batch_size))
assert torch.all(global_sigmas >= 0.1)
assert torch.all(global_sigmas < 1.0)
def test_rank_noise_is_independent_but_exactly_reproducible():
rank0, rank1 = _samplers(world_size=2)
noise0 = rank0.sample((16, 4, 3), device="cpu").noise
noise1 = rank1.sample((16, 4, 3), device="cpu").noise
replay0 = _samplers(world_size=2)[0].sample((16, 4, 3), device="cpu").noise
assert not torch.equal(noise0, noise1)
assert torch.equal(noise0, replay0)
def test_fixed_delay_matches_deployment_value_on_every_rank():
samplers = _samplers(
world_size=4,
delay=DelaySamplingConfig.fixed(deployment_delay=9),
)
delays = torch.cat(
[sampler.sample((7, 2), device="cpu").delay for sampler in samplers]
)
assert torch.equal(delays, torch.full((28,), 9, dtype=torch.long))
def test_explicit_delay_distribution_is_balanced_over_global_batch():
delay = DelaySamplingConfig.balanced(
values=(0, 5, 10),
probabilities=(0.25, 0.5, 0.25),
)
samplers = _samplers(world_size=4, delay=delay)
delays = torch.cat(
[sampler.sample((8, 2), device="cpu").delay for sampler in samplers]
)
assert Counter(delays.tolist()) == Counter({0: 8, 5: 16, 10: 8})
# The deterministic permutation should not leave categories as contiguous
# blocks even though systematic sampling creates them in CDF order first.
assert len(torch.unique(delays[:8])) > 1
def test_checkpoint_state_reproduces_the_exact_next_microbatch():
delay = DelaySamplingConfig.uniform(max_delay=11)
sampler = DistributedFlowRandomSampler(
seed=17,
rank=2,
world_size=4,
delay=delay,
)
sampler.sample((6, 5, 3), device="cpu")
checkpoint = sampler.state_dict()
expected = sampler.sample((6, 5, 3), device="cpu")
resumed = DistributedFlowRandomSampler(
seed=17,
rank=2,
world_size=4,
delay=delay,
)
resumed.load_state_dict(checkpoint)
actual = resumed.sample((6, 5, 3), device="cpu")
assert actual.draw_index == expected.draw_index == 1
assert torch.equal(actual.sigmas, expected.sigmas)
assert torch.equal(actual.delay, expected.delay)
assert torch.equal(actual.noise, expected.noise)
assert resumed.draw_index == sampler.draw_index == 2
def test_component_streams_do_not_shift_when_delay_policy_changes():
without_rtc = DistributedFlowRandomSampler(seed=31, rank=1, world_size=2)
with_rtc = DistributedFlowRandomSampler(
seed=31,
rank=1,
world_size=2,
delay=DelaySamplingConfig.uniform(max_delay=11),
)
plain = without_rtc.sample((9, 4, 3), device="cpu")
rtc = with_rtc.sample((9, 4, 3), device="cpu")
assert torch.equal(plain.sigmas, rtc.sigmas)
assert torch.equal(plain.noise, rtc.noise)
def test_resume_rejects_silent_sampling_configuration_changes():
# The production default is now sigma_min=0.05. Keep the old value
# explicit here so this test still exercises a real resume mismatch.
original = DistributedFlowRandomSampler(
seed=11, rank=0, world_size=2, sigma_min=0.1
)
changed = DistributedFlowRandomSampler(
seed=11,
rank=0,
world_size=2,
sigma_min=0.05,
)
with pytest.raises(ValueError, match="configuration changed"):
changed.load_state_dict(original.state_dict())
@pytest.mark.parametrize(
"config",
[
lambda: DelaySamplingConfig.fixed(-1),
lambda: DelaySamplingConfig.balanced((), ()),
lambda: DelaySamplingConfig.balanced((0, 1), (1.0,)),
lambda: DelaySamplingConfig.balanced((0, 0), (0.5, 0.5)),
lambda: DelaySamplingConfig.balanced((0, 1), (0.0, 0.0)),
],
)
def test_invalid_delay_config_is_rejected(config):
with pytest.raises(ValueError):
config()