File size: 4,992 Bytes
06f9561
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
from collections import Counter

import pytest
import torch

from psi.trainers.distributed_flow_sampling import (
    DelaySamplingConfig,
    DistributedFlowRandomSampler,
)


def _samplers(
    *,
    world_size: int,
    seed: int = 292285,
    delay: DelaySamplingConfig | None = None,
):
    return [
        DistributedFlowRandomSampler(
            seed=seed,
            rank=rank,
            world_size=world_size,
            sigma_min=0.1,
            sigma_max=1.0,
            delay=delay,
        )
        for rank in range(world_size)
    ]


def test_sigma_has_one_jittered_sample_per_global_stratum():
    world_size = 4
    local_batch_size = 8
    outputs = [
        sampler.sample((local_batch_size, 3, 2), device="cpu").sigmas
        for sampler in _samplers(world_size=world_size)
    ]
    global_sigmas = torch.cat(outputs)
    global_batch_size = world_size * local_batch_size

    unit_sigmas = (global_sigmas.double() - 0.1) / 0.9
    occupied_strata = torch.floor(unit_sigmas * global_batch_size).long().sort().values
    assert torch.equal(occupied_strata, torch.arange(global_batch_size))
    assert torch.all(global_sigmas >= 0.1)
    assert torch.all(global_sigmas < 1.0)


def test_rank_noise_is_independent_but_exactly_reproducible():
    rank0, rank1 = _samplers(world_size=2)
    noise0 = rank0.sample((16, 4, 3), device="cpu").noise
    noise1 = rank1.sample((16, 4, 3), device="cpu").noise
    replay0 = _samplers(world_size=2)[0].sample((16, 4, 3), device="cpu").noise

    assert not torch.equal(noise0, noise1)
    assert torch.equal(noise0, replay0)


def test_fixed_delay_matches_deployment_value_on_every_rank():
    samplers = _samplers(
        world_size=4,
        delay=DelaySamplingConfig.fixed(deployment_delay=9),
    )
    delays = torch.cat(
        [sampler.sample((7, 2), device="cpu").delay for sampler in samplers]
    )
    assert torch.equal(delays, torch.full((28,), 9, dtype=torch.long))


def test_explicit_delay_distribution_is_balanced_over_global_batch():
    delay = DelaySamplingConfig.balanced(
        values=(0, 5, 10),
        probabilities=(0.25, 0.5, 0.25),
    )
    samplers = _samplers(world_size=4, delay=delay)
    delays = torch.cat(
        [sampler.sample((8, 2), device="cpu").delay for sampler in samplers]
    )

    assert Counter(delays.tolist()) == Counter({0: 8, 5: 16, 10: 8})
    # The deterministic permutation should not leave categories as contiguous
    # blocks even though systematic sampling creates them in CDF order first.
    assert len(torch.unique(delays[:8])) > 1


def test_checkpoint_state_reproduces_the_exact_next_microbatch():
    delay = DelaySamplingConfig.uniform(max_delay=11)
    sampler = DistributedFlowRandomSampler(
        seed=17,
        rank=2,
        world_size=4,
        delay=delay,
    )
    sampler.sample((6, 5, 3), device="cpu")
    checkpoint = sampler.state_dict()
    expected = sampler.sample((6, 5, 3), device="cpu")

    resumed = DistributedFlowRandomSampler(
        seed=17,
        rank=2,
        world_size=4,
        delay=delay,
    )
    resumed.load_state_dict(checkpoint)
    actual = resumed.sample((6, 5, 3), device="cpu")

    assert actual.draw_index == expected.draw_index == 1
    assert torch.equal(actual.sigmas, expected.sigmas)
    assert torch.equal(actual.delay, expected.delay)
    assert torch.equal(actual.noise, expected.noise)
    assert resumed.draw_index == sampler.draw_index == 2


def test_component_streams_do_not_shift_when_delay_policy_changes():
    without_rtc = DistributedFlowRandomSampler(seed=31, rank=1, world_size=2)
    with_rtc = DistributedFlowRandomSampler(
        seed=31,
        rank=1,
        world_size=2,
        delay=DelaySamplingConfig.uniform(max_delay=11),
    )

    plain = without_rtc.sample((9, 4, 3), device="cpu")
    rtc = with_rtc.sample((9, 4, 3), device="cpu")
    assert torch.equal(plain.sigmas, rtc.sigmas)
    assert torch.equal(plain.noise, rtc.noise)


def test_resume_rejects_silent_sampling_configuration_changes():
    # The production default is now sigma_min=0.05.  Keep the old value
    # explicit here so this test still exercises a real resume mismatch.
    original = DistributedFlowRandomSampler(
        seed=11, rank=0, world_size=2, sigma_min=0.1
    )
    changed = DistributedFlowRandomSampler(
        seed=11,
        rank=0,
        world_size=2,
        sigma_min=0.05,
    )
    with pytest.raises(ValueError, match="configuration changed"):
        changed.load_state_dict(original.state_dict())


@pytest.mark.parametrize(
    "config",
    [
        lambda: DelaySamplingConfig.fixed(-1),
        lambda: DelaySamplingConfig.balanced((), ()),
        lambda: DelaySamplingConfig.balanced((0, 1), (1.0,)),
        lambda: DelaySamplingConfig.balanced((0, 0), (0.5, 0.5)),
        lambda: DelaySamplingConfig.balanced((0, 1), (0.0, 0.0)),
    ],
)
def test_invalid_delay_config_is_rejected(config):
    with pytest.raises(ValueError):
        config()