File size: 4,992 Bytes
06f9561 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 | from collections import Counter
import pytest
import torch
from psi.trainers.distributed_flow_sampling import (
DelaySamplingConfig,
DistributedFlowRandomSampler,
)
def _samplers(
*,
world_size: int,
seed: int = 292285,
delay: DelaySamplingConfig | None = None,
):
return [
DistributedFlowRandomSampler(
seed=seed,
rank=rank,
world_size=world_size,
sigma_min=0.1,
sigma_max=1.0,
delay=delay,
)
for rank in range(world_size)
]
def test_sigma_has_one_jittered_sample_per_global_stratum():
world_size = 4
local_batch_size = 8
outputs = [
sampler.sample((local_batch_size, 3, 2), device="cpu").sigmas
for sampler in _samplers(world_size=world_size)
]
global_sigmas = torch.cat(outputs)
global_batch_size = world_size * local_batch_size
unit_sigmas = (global_sigmas.double() - 0.1) / 0.9
occupied_strata = torch.floor(unit_sigmas * global_batch_size).long().sort().values
assert torch.equal(occupied_strata, torch.arange(global_batch_size))
assert torch.all(global_sigmas >= 0.1)
assert torch.all(global_sigmas < 1.0)
def test_rank_noise_is_independent_but_exactly_reproducible():
rank0, rank1 = _samplers(world_size=2)
noise0 = rank0.sample((16, 4, 3), device="cpu").noise
noise1 = rank1.sample((16, 4, 3), device="cpu").noise
replay0 = _samplers(world_size=2)[0].sample((16, 4, 3), device="cpu").noise
assert not torch.equal(noise0, noise1)
assert torch.equal(noise0, replay0)
def test_fixed_delay_matches_deployment_value_on_every_rank():
samplers = _samplers(
world_size=4,
delay=DelaySamplingConfig.fixed(deployment_delay=9),
)
delays = torch.cat(
[sampler.sample((7, 2), device="cpu").delay for sampler in samplers]
)
assert torch.equal(delays, torch.full((28,), 9, dtype=torch.long))
def test_explicit_delay_distribution_is_balanced_over_global_batch():
delay = DelaySamplingConfig.balanced(
values=(0, 5, 10),
probabilities=(0.25, 0.5, 0.25),
)
samplers = _samplers(world_size=4, delay=delay)
delays = torch.cat(
[sampler.sample((8, 2), device="cpu").delay for sampler in samplers]
)
assert Counter(delays.tolist()) == Counter({0: 8, 5: 16, 10: 8})
# The deterministic permutation should not leave categories as contiguous
# blocks even though systematic sampling creates them in CDF order first.
assert len(torch.unique(delays[:8])) > 1
def test_checkpoint_state_reproduces_the_exact_next_microbatch():
delay = DelaySamplingConfig.uniform(max_delay=11)
sampler = DistributedFlowRandomSampler(
seed=17,
rank=2,
world_size=4,
delay=delay,
)
sampler.sample((6, 5, 3), device="cpu")
checkpoint = sampler.state_dict()
expected = sampler.sample((6, 5, 3), device="cpu")
resumed = DistributedFlowRandomSampler(
seed=17,
rank=2,
world_size=4,
delay=delay,
)
resumed.load_state_dict(checkpoint)
actual = resumed.sample((6, 5, 3), device="cpu")
assert actual.draw_index == expected.draw_index == 1
assert torch.equal(actual.sigmas, expected.sigmas)
assert torch.equal(actual.delay, expected.delay)
assert torch.equal(actual.noise, expected.noise)
assert resumed.draw_index == sampler.draw_index == 2
def test_component_streams_do_not_shift_when_delay_policy_changes():
without_rtc = DistributedFlowRandomSampler(seed=31, rank=1, world_size=2)
with_rtc = DistributedFlowRandomSampler(
seed=31,
rank=1,
world_size=2,
delay=DelaySamplingConfig.uniform(max_delay=11),
)
plain = without_rtc.sample((9, 4, 3), device="cpu")
rtc = with_rtc.sample((9, 4, 3), device="cpu")
assert torch.equal(plain.sigmas, rtc.sigmas)
assert torch.equal(plain.noise, rtc.noise)
def test_resume_rejects_silent_sampling_configuration_changes():
# The production default is now sigma_min=0.05. Keep the old value
# explicit here so this test still exercises a real resume mismatch.
original = DistributedFlowRandomSampler(
seed=11, rank=0, world_size=2, sigma_min=0.1
)
changed = DistributedFlowRandomSampler(
seed=11,
rank=0,
world_size=2,
sigma_min=0.05,
)
with pytest.raises(ValueError, match="configuration changed"):
changed.load_state_dict(original.state_dict())
@pytest.mark.parametrize(
"config",
[
lambda: DelaySamplingConfig.fixed(-1),
lambda: DelaySamplingConfig.balanced((), ()),
lambda: DelaySamplingConfig.balanced((0, 1), (1.0,)),
lambda: DelaySamplingConfig.balanced((0, 0), (0.5, 0.5)),
lambda: DelaySamplingConfig.balanced((0, 1), (0.0, 0.0)),
],
)
def test_invalid_delay_config_is_rejected(config):
with pytest.raises(ValueError):
config()
|