Naphula commited on
Commit
e7a095d
·
verified ·
1 Parent(s): 3ceb796

Upload 2 files

Browse files
Files changed (2) hide show
  1. sparsify3.py +207 -0
  2. sparsify_v3_notes.md +35 -0
sparsify3.py ADDED
@@ -0,0 +1,207 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (C) 2025 Arcee AI
2
+ # SPDX-License-Identifier: LGPL-3.0-only
3
+
4
+ from enum import Enum
5
+ from typing import Optional
6
+
7
+ import torch
8
+
9
+
10
+ class SparsificationMethod(str, Enum):
11
+ magnitude = "magnitude"
12
+ random = "random"
13
+ magnitude_outliers = "magnitude_outliers"
14
+ della_magprune = "della_magprune"
15
+
16
+
17
+ class RescaleNorm(str, Enum):
18
+ l1 = "l1"
19
+ l2 = "l2"
20
+ linf = "linf"
21
+
22
+
23
+ def rescaled_masked_tensor(
24
+ tensor: torch.Tensor,
25
+ mask: torch.Tensor,
26
+ norm: Optional[RescaleNorm],
27
+ eps: float = 1e-7,
28
+ ) -> torch.Tensor:
29
+ """Apply a mask to a tensor and rescale to match the original tensor norm.
30
+
31
+ Args:
32
+ tensor (torch.Tensor): Input tensor.
33
+ mask (torch.Tensor): Mask to apply.
34
+ norm (RescaleNorm): Which norm to match (l1, l2, linf).
35
+ eps (float): Tolerance for small norms to avoid division by zero.
36
+ """
37
+ masked = tensor * mask
38
+ if norm is None:
39
+ return masked
40
+ elif norm == RescaleNorm.l1:
41
+ before_scale = tensor.abs().sum()
42
+ after_scale = masked.abs().sum()
43
+ elif norm == RescaleNorm.l2:
44
+ before_scale = tensor.norm()
45
+ after_scale = masked.norm()
46
+ elif norm == RescaleNorm.linf:
47
+ before_scale = tensor.abs().max()
48
+ after_scale = masked.abs().max()
49
+ else:
50
+ raise NotImplementedError(norm)
51
+ if before_scale < eps or after_scale < eps:
52
+ return masked
53
+ return masked * (before_scale / after_scale)
54
+
55
+
56
+ def magnitude(
57
+ tensor: torch.Tensor, density: float, rescale_norm: Optional[RescaleNorm] = None
58
+ ) -> torch.Tensor:
59
+ """Masks out the smallest values, retaining a proportion of `density`."""
60
+ if density >= 1:
61
+ return tensor
62
+
63
+ k = int(density * tensor.numel())
64
+
65
+ assert k > 0, "not gonna zero out the whole tensor buddy"
66
+ mask = torch.zeros_like(tensor)
67
+ w = tensor.abs().view(-1)
68
+ if w.device.type == "cpu":
69
+ w = w.float()
70
+ topk = torch.argsort(w, descending=True)[:k]
71
+ mask.view(-1)[topk] = 1
72
+
73
+ res = rescaled_masked_tensor(tensor, mask, rescale_norm)
74
+ return res
75
+
76
+
77
+ def magnitude_outliers(
78
+ tensor: torch.Tensor,
79
+ density: float,
80
+ rescale_norm: Optional[RescaleNorm] = None,
81
+ gamma: float = 0.01,
82
+ ):
83
+ """Masks out smallest values in addition to large outliers.
84
+
85
+ The `gamma` proportion of the largest weights are first removed, then the
86
+ smallest weights are removed to achieve the desired density.
87
+
88
+ Args:
89
+ tensor (torch.Tensor): The tensor to sparsify.
90
+ density (float): The proportion of weights to retain.
91
+ gamma (float): Percent of largest weights to remove.
92
+ """
93
+ if density >= 1:
94
+ return tensor
95
+
96
+ num_elems = tensor.numel()
97
+ target_n = int(density * num_elems)
98
+ n_top = int(gamma * num_elems)
99
+ n_bot = num_elems - target_n - n_top
100
+
101
+ if n_bot < 0:
102
+ # cut down on the number of large weights to remove in
103
+ # order to hit the target density
104
+ n_top += n_bot
105
+ n_bot = 0
106
+
107
+ w = tensor.abs().view(-1)
108
+ if w.device.type == "cpu":
109
+ w = w.float()
110
+ indices = torch.sort(w, descending=False).indices
111
+ mask = torch.zeros_like(tensor)
112
+
113
+ mask.view(-1)[indices[n_bot:-n_top]] = 1
114
+
115
+ res = rescaled_masked_tensor(tensor, mask, rescale_norm)
116
+ return res
117
+
118
+
119
+ def bernoulli(
120
+ tensor: torch.Tensor, density: float, rescale_norm: Optional[RescaleNorm] = None
121
+ ) -> torch.Tensor:
122
+ if density >= 1:
123
+ return tensor
124
+
125
+ if (tensor.device.type != "cpu") or tensor.dtype == torch.bfloat16:
126
+ work_dtype = tensor.dtype
127
+ else:
128
+ # torch.bernoulli not implemented for float16 on CPU, upcast to float32
129
+ work_dtype = torch.float32
130
+
131
+ mask = torch.bernoulli(
132
+ torch.full_like(input=tensor, fill_value=density, dtype=work_dtype)
133
+ )
134
+ res = rescaled_masked_tensor(tensor.to(work_dtype), mask, rescale_norm)
135
+ return res.to(tensor.dtype)
136
+
137
+
138
+ def della_magprune(
139
+ tensor: torch.Tensor,
140
+ density: float,
141
+ epsilon: float,
142
+ rescale_norm: Optional[RescaleNorm] = None,
143
+ ) -> torch.Tensor:
144
+ if density >= 1:
145
+ return tensor
146
+ if density <= 0:
147
+ return torch.zeros_like(tensor)
148
+
149
+ # --- SAFETY GUARD START ---
150
+ # Ensure density isn't exactly 0 or 1
151
+ density = max(1e-4, min(1.0 - 1e-4, density))
152
+
153
+ # Epsilon must be < density AND < (1 - density)
154
+ # If the optimizer guessed a bad epsilon, we shrink it to the max allowed value
155
+ max_epsilon = min(density, 1.0 - density) - 1e-4
156
+ if abs(epsilon) > max_epsilon:
157
+ epsilon = max_epsilon if epsilon > 0 else -max_epsilon
158
+ # --- SAFETY GUARD END ---
159
+
160
+ orig_shape = tensor.shape
161
+ work_dtype = (
162
+ tensor.dtype
163
+ if tensor.device.type != "cpu" or tensor.dtype == torch.bfloat16
164
+ else torch.float32
165
+ )
166
+
167
+ if len(tensor.shape) < 2:
168
+ tensor = tensor.unsqueeze(0)
169
+ magnitudes = tensor.abs()
170
+
171
+ sorted_indices = torch.argsort(magnitudes, dim=1, descending=False)
172
+ ranks = sorted_indices.argsort(dim=1).to(work_dtype) + 1
173
+
174
+ min_ranks = ranks.min(dim=1, keepdim=True).values
175
+ max_ranks = ranks.max(dim=1, keepdim=True).values
176
+ rank_norm = ((ranks - min_ranks) / (max_ranks - min_ranks).clamp(min=1e-8)).clamp(0, 1)
177
+
178
+ # Now this line is guaranteed not to produce values < 0 or > 1
179
+ probs = (density - epsilon) + rank_norm * 2 * epsilon
180
+ mask = torch.bernoulli(probs.clamp(0, 1)).to(work_dtype)
181
+
182
+ res = rescaled_masked_tensor(tensor.to(work_dtype), mask, rescale_norm)
183
+ return res.to(tensor.dtype).reshape(orig_shape)
184
+
185
+
186
+ def sparsify(
187
+ tensor: torch.Tensor,
188
+ density: float,
189
+ method: SparsificationMethod,
190
+ gamma: float = 0,
191
+ epsilon: float = 0,
192
+ rescale_norm: Optional[RescaleNorm] = None,
193
+ ) -> torch.Tensor:
194
+ if method == SparsificationMethod.magnitude:
195
+ return magnitude(tensor, density=density, rescale_norm=rescale_norm)
196
+ elif method == SparsificationMethod.random:
197
+ return bernoulli(tensor, density=density, rescale_norm=rescale_norm)
198
+ elif method == SparsificationMethod.magnitude_outliers:
199
+ return magnitude_outliers(
200
+ tensor, density=density, rescale_norm=rescale_norm, gamma=gamma
201
+ )
202
+ elif method == SparsificationMethod.della_magprune:
203
+ return della_magprune(
204
+ tensor, density=density, epsilon=epsilon, rescale_norm=rescale_norm
205
+ )
206
+ else:
207
+ raise NotImplementedError(method)
sparsify_v3_notes.md ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ The two scripts you provided are nearly identical in structure, but the second script contains significant **safety enhancements** and **robustness fixes** for the `della_magprune` method.
2
+
3
+ Here are the specific differences:
4
+
5
+ ### 1. Parameter Validation (Safety Guards)
6
+ The second script adds a "Safety Guard" block at the start of the `della_magprune` function. This prevents the function from crashing or producing invalid results if the input parameters (`density` or `epsilon`) are mathematically impossible.
7
+ * **Density Clipping:** It ensures `density` stays within a valid range (between `1e-4` and `1-1e-4`).
8
+ * **Epsilon Adjustment:** It automatically shrinks `epsilon` if it is too large. Since the algorithm calculates probabilities as `density +/- epsilon`, an epsilon that is too large would result in probabilities greater than 1 or less than 0. The second script forces epsilon to be within a safe bound.
9
+
10
+ ### 2. Division by Zero Protection
11
+ In the rank normalization step of `della_magprune`:
12
+ * **Script 1:** `rank_norm = ((ranks - min_ranks) / (max_ranks - min_ranks))`
13
+ * **Script 2:** `rank_norm = ((ranks - min_ranks) / (max_ranks - min_ranks).clamp(min=1e-8))`
14
+ * **Impact:** If a tensor has only one unique value (meaning `max_ranks == min_ranks`), the first script would divide by zero and produce `NaN` values. The second script uses `.clamp(min=1e-8)` to ensure the denominator is never zero.
15
+
16
+ ### 3. Probability Clipping
17
+ In the final step of generating the mask:
18
+ * **Script 1:** `probs = (density - epsilon) + rank_norm * 2 * epsilon`
19
+ * **Script 2:** `probs = (density - epsilon) + rank_norm * 2 * epsilon` followed by `torch.bernoulli(probs.clamp(0, 1))`
20
+ * **Impact:** Even with the epsilon guards, floating-point errors could theoretically push a probability slightly outside the $[0, 1]$ range. The second script adds `.clamp(0, 1)` to the Bernoulli input to ensure PyTorch does not throw an error.
21
+
22
+ ### 4. Logic Flow in `della_magprune`
23
+ * **Script 1** has a check: `if density + epsilon >= 1 or density - epsilon <= 0: raise ValueError(...)`. This causes the script to **crash** if the parameters are bad.
24
+ * **Script 2** removes that `ValueError` and replaces it with the "Safety Guard" logic mentioned above. Instead of crashing, it **corrects** the values and continues running.
25
+
26
+ ### Summary Table
27
+
28
+ | Feature | Script 1 | Script 2 |
29
+ | :--- | :--- | :--- |
30
+ | **Bad Inputs** | Crashes with `ValueError` | Automatically fixes/clips values |
31
+ | **Single-value Tensors** | May produce `NaN` (Div by 0) | Safe (Clamped denominator) |
32
+ | **Bernoulli Stability** | Risk of out-of-bounds error | Guaranteed $[0, 1]$ range |
33
+ | **Reliability** | Experimental/Strict | Production-ready/Robust |
34
+
35
+ **Recommendation:** Use the **second script**. It is a more mature version of the code designed to handle edge cases and prevent runtime failures during automated optimization or training.