Spaces:
Running
Running
Upload 2 files
Browse files- sparsify3.py +207 -0
- sparsify_v3_notes.md +35 -0
sparsify3.py
ADDED
|
@@ -0,0 +1,207 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (C) 2025 Arcee AI
|
| 2 |
+
# SPDX-License-Identifier: LGPL-3.0-only
|
| 3 |
+
|
| 4 |
+
from enum import Enum
|
| 5 |
+
from typing import Optional
|
| 6 |
+
|
| 7 |
+
import torch
|
| 8 |
+
|
| 9 |
+
|
| 10 |
+
class SparsificationMethod(str, Enum):
|
| 11 |
+
magnitude = "magnitude"
|
| 12 |
+
random = "random"
|
| 13 |
+
magnitude_outliers = "magnitude_outliers"
|
| 14 |
+
della_magprune = "della_magprune"
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
class RescaleNorm(str, Enum):
|
| 18 |
+
l1 = "l1"
|
| 19 |
+
l2 = "l2"
|
| 20 |
+
linf = "linf"
|
| 21 |
+
|
| 22 |
+
|
| 23 |
+
def rescaled_masked_tensor(
|
| 24 |
+
tensor: torch.Tensor,
|
| 25 |
+
mask: torch.Tensor,
|
| 26 |
+
norm: Optional[RescaleNorm],
|
| 27 |
+
eps: float = 1e-7,
|
| 28 |
+
) -> torch.Tensor:
|
| 29 |
+
"""Apply a mask to a tensor and rescale to match the original tensor norm.
|
| 30 |
+
|
| 31 |
+
Args:
|
| 32 |
+
tensor (torch.Tensor): Input tensor.
|
| 33 |
+
mask (torch.Tensor): Mask to apply.
|
| 34 |
+
norm (RescaleNorm): Which norm to match (l1, l2, linf).
|
| 35 |
+
eps (float): Tolerance for small norms to avoid division by zero.
|
| 36 |
+
"""
|
| 37 |
+
masked = tensor * mask
|
| 38 |
+
if norm is None:
|
| 39 |
+
return masked
|
| 40 |
+
elif norm == RescaleNorm.l1:
|
| 41 |
+
before_scale = tensor.abs().sum()
|
| 42 |
+
after_scale = masked.abs().sum()
|
| 43 |
+
elif norm == RescaleNorm.l2:
|
| 44 |
+
before_scale = tensor.norm()
|
| 45 |
+
after_scale = masked.norm()
|
| 46 |
+
elif norm == RescaleNorm.linf:
|
| 47 |
+
before_scale = tensor.abs().max()
|
| 48 |
+
after_scale = masked.abs().max()
|
| 49 |
+
else:
|
| 50 |
+
raise NotImplementedError(norm)
|
| 51 |
+
if before_scale < eps or after_scale < eps:
|
| 52 |
+
return masked
|
| 53 |
+
return masked * (before_scale / after_scale)
|
| 54 |
+
|
| 55 |
+
|
| 56 |
+
def magnitude(
|
| 57 |
+
tensor: torch.Tensor, density: float, rescale_norm: Optional[RescaleNorm] = None
|
| 58 |
+
) -> torch.Tensor:
|
| 59 |
+
"""Masks out the smallest values, retaining a proportion of `density`."""
|
| 60 |
+
if density >= 1:
|
| 61 |
+
return tensor
|
| 62 |
+
|
| 63 |
+
k = int(density * tensor.numel())
|
| 64 |
+
|
| 65 |
+
assert k > 0, "not gonna zero out the whole tensor buddy"
|
| 66 |
+
mask = torch.zeros_like(tensor)
|
| 67 |
+
w = tensor.abs().view(-1)
|
| 68 |
+
if w.device.type == "cpu":
|
| 69 |
+
w = w.float()
|
| 70 |
+
topk = torch.argsort(w, descending=True)[:k]
|
| 71 |
+
mask.view(-1)[topk] = 1
|
| 72 |
+
|
| 73 |
+
res = rescaled_masked_tensor(tensor, mask, rescale_norm)
|
| 74 |
+
return res
|
| 75 |
+
|
| 76 |
+
|
| 77 |
+
def magnitude_outliers(
|
| 78 |
+
tensor: torch.Tensor,
|
| 79 |
+
density: float,
|
| 80 |
+
rescale_norm: Optional[RescaleNorm] = None,
|
| 81 |
+
gamma: float = 0.01,
|
| 82 |
+
):
|
| 83 |
+
"""Masks out smallest values in addition to large outliers.
|
| 84 |
+
|
| 85 |
+
The `gamma` proportion of the largest weights are first removed, then the
|
| 86 |
+
smallest weights are removed to achieve the desired density.
|
| 87 |
+
|
| 88 |
+
Args:
|
| 89 |
+
tensor (torch.Tensor): The tensor to sparsify.
|
| 90 |
+
density (float): The proportion of weights to retain.
|
| 91 |
+
gamma (float): Percent of largest weights to remove.
|
| 92 |
+
"""
|
| 93 |
+
if density >= 1:
|
| 94 |
+
return tensor
|
| 95 |
+
|
| 96 |
+
num_elems = tensor.numel()
|
| 97 |
+
target_n = int(density * num_elems)
|
| 98 |
+
n_top = int(gamma * num_elems)
|
| 99 |
+
n_bot = num_elems - target_n - n_top
|
| 100 |
+
|
| 101 |
+
if n_bot < 0:
|
| 102 |
+
# cut down on the number of large weights to remove in
|
| 103 |
+
# order to hit the target density
|
| 104 |
+
n_top += n_bot
|
| 105 |
+
n_bot = 0
|
| 106 |
+
|
| 107 |
+
w = tensor.abs().view(-1)
|
| 108 |
+
if w.device.type == "cpu":
|
| 109 |
+
w = w.float()
|
| 110 |
+
indices = torch.sort(w, descending=False).indices
|
| 111 |
+
mask = torch.zeros_like(tensor)
|
| 112 |
+
|
| 113 |
+
mask.view(-1)[indices[n_bot:-n_top]] = 1
|
| 114 |
+
|
| 115 |
+
res = rescaled_masked_tensor(tensor, mask, rescale_norm)
|
| 116 |
+
return res
|
| 117 |
+
|
| 118 |
+
|
| 119 |
+
def bernoulli(
|
| 120 |
+
tensor: torch.Tensor, density: float, rescale_norm: Optional[RescaleNorm] = None
|
| 121 |
+
) -> torch.Tensor:
|
| 122 |
+
if density >= 1:
|
| 123 |
+
return tensor
|
| 124 |
+
|
| 125 |
+
if (tensor.device.type != "cpu") or tensor.dtype == torch.bfloat16:
|
| 126 |
+
work_dtype = tensor.dtype
|
| 127 |
+
else:
|
| 128 |
+
# torch.bernoulli not implemented for float16 on CPU, upcast to float32
|
| 129 |
+
work_dtype = torch.float32
|
| 130 |
+
|
| 131 |
+
mask = torch.bernoulli(
|
| 132 |
+
torch.full_like(input=tensor, fill_value=density, dtype=work_dtype)
|
| 133 |
+
)
|
| 134 |
+
res = rescaled_masked_tensor(tensor.to(work_dtype), mask, rescale_norm)
|
| 135 |
+
return res.to(tensor.dtype)
|
| 136 |
+
|
| 137 |
+
|
| 138 |
+
def della_magprune(
|
| 139 |
+
tensor: torch.Tensor,
|
| 140 |
+
density: float,
|
| 141 |
+
epsilon: float,
|
| 142 |
+
rescale_norm: Optional[RescaleNorm] = None,
|
| 143 |
+
) -> torch.Tensor:
|
| 144 |
+
if density >= 1:
|
| 145 |
+
return tensor
|
| 146 |
+
if density <= 0:
|
| 147 |
+
return torch.zeros_like(tensor)
|
| 148 |
+
|
| 149 |
+
# --- SAFETY GUARD START ---
|
| 150 |
+
# Ensure density isn't exactly 0 or 1
|
| 151 |
+
density = max(1e-4, min(1.0 - 1e-4, density))
|
| 152 |
+
|
| 153 |
+
# Epsilon must be < density AND < (1 - density)
|
| 154 |
+
# If the optimizer guessed a bad epsilon, we shrink it to the max allowed value
|
| 155 |
+
max_epsilon = min(density, 1.0 - density) - 1e-4
|
| 156 |
+
if abs(epsilon) > max_epsilon:
|
| 157 |
+
epsilon = max_epsilon if epsilon > 0 else -max_epsilon
|
| 158 |
+
# --- SAFETY GUARD END ---
|
| 159 |
+
|
| 160 |
+
orig_shape = tensor.shape
|
| 161 |
+
work_dtype = (
|
| 162 |
+
tensor.dtype
|
| 163 |
+
if tensor.device.type != "cpu" or tensor.dtype == torch.bfloat16
|
| 164 |
+
else torch.float32
|
| 165 |
+
)
|
| 166 |
+
|
| 167 |
+
if len(tensor.shape) < 2:
|
| 168 |
+
tensor = tensor.unsqueeze(0)
|
| 169 |
+
magnitudes = tensor.abs()
|
| 170 |
+
|
| 171 |
+
sorted_indices = torch.argsort(magnitudes, dim=1, descending=False)
|
| 172 |
+
ranks = sorted_indices.argsort(dim=1).to(work_dtype) + 1
|
| 173 |
+
|
| 174 |
+
min_ranks = ranks.min(dim=1, keepdim=True).values
|
| 175 |
+
max_ranks = ranks.max(dim=1, keepdim=True).values
|
| 176 |
+
rank_norm = ((ranks - min_ranks) / (max_ranks - min_ranks).clamp(min=1e-8)).clamp(0, 1)
|
| 177 |
+
|
| 178 |
+
# Now this line is guaranteed not to produce values < 0 or > 1
|
| 179 |
+
probs = (density - epsilon) + rank_norm * 2 * epsilon
|
| 180 |
+
mask = torch.bernoulli(probs.clamp(0, 1)).to(work_dtype)
|
| 181 |
+
|
| 182 |
+
res = rescaled_masked_tensor(tensor.to(work_dtype), mask, rescale_norm)
|
| 183 |
+
return res.to(tensor.dtype).reshape(orig_shape)
|
| 184 |
+
|
| 185 |
+
|
| 186 |
+
def sparsify(
|
| 187 |
+
tensor: torch.Tensor,
|
| 188 |
+
density: float,
|
| 189 |
+
method: SparsificationMethod,
|
| 190 |
+
gamma: float = 0,
|
| 191 |
+
epsilon: float = 0,
|
| 192 |
+
rescale_norm: Optional[RescaleNorm] = None,
|
| 193 |
+
) -> torch.Tensor:
|
| 194 |
+
if method == SparsificationMethod.magnitude:
|
| 195 |
+
return magnitude(tensor, density=density, rescale_norm=rescale_norm)
|
| 196 |
+
elif method == SparsificationMethod.random:
|
| 197 |
+
return bernoulli(tensor, density=density, rescale_norm=rescale_norm)
|
| 198 |
+
elif method == SparsificationMethod.magnitude_outliers:
|
| 199 |
+
return magnitude_outliers(
|
| 200 |
+
tensor, density=density, rescale_norm=rescale_norm, gamma=gamma
|
| 201 |
+
)
|
| 202 |
+
elif method == SparsificationMethod.della_magprune:
|
| 203 |
+
return della_magprune(
|
| 204 |
+
tensor, density=density, epsilon=epsilon, rescale_norm=rescale_norm
|
| 205 |
+
)
|
| 206 |
+
else:
|
| 207 |
+
raise NotImplementedError(method)
|
sparsify_v3_notes.md
ADDED
|
@@ -0,0 +1,35 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
The two scripts you provided are nearly identical in structure, but the second script contains significant **safety enhancements** and **robustness fixes** for the `della_magprune` method.
|
| 2 |
+
|
| 3 |
+
Here are the specific differences:
|
| 4 |
+
|
| 5 |
+
### 1. Parameter Validation (Safety Guards)
|
| 6 |
+
The second script adds a "Safety Guard" block at the start of the `della_magprune` function. This prevents the function from crashing or producing invalid results if the input parameters (`density` or `epsilon`) are mathematically impossible.
|
| 7 |
+
* **Density Clipping:** It ensures `density` stays within a valid range (between `1e-4` and `1-1e-4`).
|
| 8 |
+
* **Epsilon Adjustment:** It automatically shrinks `epsilon` if it is too large. Since the algorithm calculates probabilities as `density +/- epsilon`, an epsilon that is too large would result in probabilities greater than 1 or less than 0. The second script forces epsilon to be within a safe bound.
|
| 9 |
+
|
| 10 |
+
### 2. Division by Zero Protection
|
| 11 |
+
In the rank normalization step of `della_magprune`:
|
| 12 |
+
* **Script 1:** `rank_norm = ((ranks - min_ranks) / (max_ranks - min_ranks))`
|
| 13 |
+
* **Script 2:** `rank_norm = ((ranks - min_ranks) / (max_ranks - min_ranks).clamp(min=1e-8))`
|
| 14 |
+
* **Impact:** If a tensor has only one unique value (meaning `max_ranks == min_ranks`), the first script would divide by zero and produce `NaN` values. The second script uses `.clamp(min=1e-8)` to ensure the denominator is never zero.
|
| 15 |
+
|
| 16 |
+
### 3. Probability Clipping
|
| 17 |
+
In the final step of generating the mask:
|
| 18 |
+
* **Script 1:** `probs = (density - epsilon) + rank_norm * 2 * epsilon`
|
| 19 |
+
* **Script 2:** `probs = (density - epsilon) + rank_norm * 2 * epsilon` followed by `torch.bernoulli(probs.clamp(0, 1))`
|
| 20 |
+
* **Impact:** Even with the epsilon guards, floating-point errors could theoretically push a probability slightly outside the $[0, 1]$ range. The second script adds `.clamp(0, 1)` to the Bernoulli input to ensure PyTorch does not throw an error.
|
| 21 |
+
|
| 22 |
+
### 4. Logic Flow in `della_magprune`
|
| 23 |
+
* **Script 1** has a check: `if density + epsilon >= 1 or density - epsilon <= 0: raise ValueError(...)`. This causes the script to **crash** if the parameters are bad.
|
| 24 |
+
* **Script 2** removes that `ValueError` and replaces it with the "Safety Guard" logic mentioned above. Instead of crashing, it **corrects** the values and continues running.
|
| 25 |
+
|
| 26 |
+
### Summary Table
|
| 27 |
+
|
| 28 |
+
| Feature | Script 1 | Script 2 |
|
| 29 |
+
| :--- | :--- | :--- |
|
| 30 |
+
| **Bad Inputs** | Crashes with `ValueError` | Automatically fixes/clips values |
|
| 31 |
+
| **Single-value Tensors** | May produce `NaN` (Div by 0) | Safe (Clamped denominator) |
|
| 32 |
+
| **Bernoulli Stability** | Risk of out-of-bounds error | Guaranteed $[0, 1]$ range |
|
| 33 |
+
| **Reliability** | Experimental/Strict | Production-ready/Robust |
|
| 34 |
+
|
| 35 |
+
**Recommendation:** Use the **second script**. It is a more mature version of the code designed to handle edge cases and prevent runtime failures during automated optimization or training.
|