File size: 5,085 Bytes
7ffed9a
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
"""
Unit tests for PDP implementation to verify correctness against paper formulas.
"""

import torch
import math
from pdp import pdp_soft_mask, compute_threshold, PDPPruner
import torch.nn as nn


def test_soft_mask_boundary_conditions():
    """
    From the paper (Section 3.1):
    - m(w=0) should be 0.5 when |w| = t (equal chance)
    - m(w=0) -> 0 when w=0 (actually z(0)=1 so m(0)=0... wait let me check)
    - m(w->inf) -> 1
    """
    tau = 1e-4
    t = 0.6

    # w = t -> equal chance, m should be 0.5
    w_eq = torch.tensor([t])
    m_eq = pdp_soft_mask(w_eq, t, tau)
    assert abs(m_eq.item() - 0.5) < 1e-3, f"m(t) should be ~0.5, got {m_eq.item()}"

    # w >> t -> m -> 1
    w_big = torch.tensor([10.0])
    m_big = pdp_soft_mask(w_big, t, tau)
    assert m_big.item() > 0.99, f"m(>>t) should be ~1, got {m_big.item()}"

    # w << t -> m -> 0
    w_small = torch.tensor([0.001])
    m_small = pdp_soft_mask(w_small, t, tau)
    assert m_small.item() < 0.01, f"m(<<t) should be ~0, got {m_small.item()}"

    print("✅ test_soft_mask_boundary_conditions passed")


def test_soft_mask_monotonicity():
    """
    Larger |w| -> larger m(w) (higher chance to keep).
    """
    tau = 1e-4
    t = 0.6
    weights = torch.linspace(0.0, 2.0, 100)
    masks = pdp_soft_mask(weights, t, tau)

    # Check monotonicity
    for i in range(len(weights) - 1):
        assert masks[i] <= masks[i + 1] + 1e-6, "m(w) must be monotonically increasing"

    print("✅ test_soft_mask_monotonicity passed")


def test_gradient_flow():
    """
    The soft mask must allow gradients to flow through.
    Paper Eq. 2: Δw = m(w)·Δŵ + (2w²/τ)·m(w)·(1-m(w))·Δŵ
    We verify this with a mild tau so values near the boundary aren't underflowed.
    """
    tau = 1e-1  # larger tau to avoid numerical underflow near boundary
    t = 0.6
    w = torch.tensor([0.59], requires_grad=True)  # very close to boundary

    # Forward: apply soft mask
    masked = pdp_soft_mask(w, t, tau) * w
    loss = masked.sum()
    loss.backward()

    # Check gradient exists and is non-zero
    assert w.grad is not None, "Gradient should flow through PDP mask"
    assert w.grad.abs().item() > 0, f"Gradient should be non-zero, got {w.grad.item()}"

    # Near boundary (m≈0.5), the extra gradient term should be maximized
    # dŵ/dw = m(w) + (2w²/τ)·m(w)·(1-m(w))
    m_val = pdp_soft_mask(w.detach(), t, tau).item()
    expected = m_val + 2 * (w.item() ** 2) / tau * m_val * (1 - m_val)
    actual = w.grad.item()
    # Allow tolerance for autograd numerical differences
    assert abs(actual - expected) < 0.1, \
        f"Expected grad ~{expected:.4f}, got {actual:.4f}"

    print("✅ test_gradient_flow passed")


def test_threshold_computation():
    """
    Test that compute_threshold yields correct sparsity.
    """
    torch.manual_seed(42)
    weights = torch.randn(1000).abs()
    sparsity = 0.3
    t = compute_threshold(weights, sparsity)

    below = (weights <= t).float().sum().item()
    actual_sparsity = below / weights.numel()
    # Should be close to target (within 1 element)
    assert abs(actual_sparsity - sparsity) < 0.01, \
        f"Threshold sparsity {actual_sparsity} far from target {sparsity}"

    print("✅ test_threshold_computation passed")


def test_pdp_pruner_end_to_end():
    """
    Full end-to-end test: model, prune, hard prune.
    """
    class Net(nn.Module):
        def __init__(self):
            super().__init__()
            self.conv1 = nn.Conv2d(3, 16, 3, padding=1)
            self.conv2 = nn.Conv2d(16, 32, 3, padding=1)
            self.fc = nn.Linear(32, 10)

        def forward(self, x):
            x = torch.relu(self.conv1(x))
            x = torch.relu(self.conv2(x))
            x = x.mean(dim=(2, 3))
            return self.fc(x)

    model = Net()
    pruner = PDPPruner(model, target_sparsity=0.5, s=2, epsilon=0.1, tau=1e-4)
    pruner.attach()

    # Simulate 4 training steps
    for epoch in range(4):
        x = torch.randn(4, 3, 8, 8)
        y = model(x)
        loss = y.sum()
        loss.backward()
        with torch.no_grad():
            for p in model.parameters():
                if p.grad is not None:
                    p -= 0.01 * p.grad
                    p.grad.zero_()
        pruner.step(epoch)

    # Hard prune
    pruner.hard_prune()
    sparsity = pruner.get_sparsity()
    assert sparsity > 0, f"After hard prune, sparsity should be > 0, got {sparsity}"

    # Check weights are actually zero
    for name, param in model.named_parameters():
        if "weight" in name and any(k in name for k in ["conv", "fc"]):
            zeros = (param.data == 0).float().sum().item()
            assert zeros > 0, f"No weights pruned in {name}"

    pruner.detach()
    print(f"✅ test_pdp_pruner_end_to_end passed (final sparsity={sparsity:.4f})")


if __name__ == "__main__":
    test_soft_mask_boundary_conditions()
    test_soft_mask_monotonicity()
    test_gradient_flow()
    test_threshold_computation()
    test_pdp_pruner_end_to_end()
    print("\n🎉 All PDP tests passed!")