Spaces:
Running
Running
sce audit was surprisingly difficult for some reason but it works now, rename to sce.py
Browse files- sce_audit.py +106 -0
sce_audit.py
ADDED
|
@@ -0,0 +1,106 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
# Copyright (C) 2025 Arcee AI
|
| 2 |
+
# SPDX-License-Identifier: LGPL-3.0-only
|
| 3 |
+
|
| 4 |
+
from typing import List, Optional
|
| 5 |
+
|
| 6 |
+
import torch
|
| 7 |
+
|
| 8 |
+
from mergekit.merge_methods.easy_define import merge_method
|
| 9 |
+
from mergekit.merge_methods.generalized_task_arithmetic import (
|
| 10 |
+
get_mask as sign_consensus_mask,
|
| 11 |
+
)
|
| 12 |
+
from mergekit.architecture import WeightInfo
|
| 13 |
+
from mergekit.common import ModelReference
|
| 14 |
+
import inspect
|
| 15 |
+
|
| 16 |
+
|
| 17 |
+
def log_sce_audit(layer_name: str, base_name: str, donor_names: List[str], variances: List[float], select_topk: float):
|
| 18 |
+
bar_char = "█"
|
| 19 |
+
lines = [f"\n[SCE Audit] Layer: {layer_name} | select_topk={select_topk}"]
|
| 20 |
+
clean_base = base_name.split("\\")[-1].split("/")[-1][:50]
|
| 21 |
+
lines.append(f" [BASE] {clean_base:<50}")
|
| 22 |
+
|
| 23 |
+
total_var = sum(variances) if sum(variances) > 0 else 1.0
|
| 24 |
+
|
| 25 |
+
for name, var in zip(donor_names, variances):
|
| 26 |
+
pct = (var / total_var) * 100
|
| 27 |
+
bar_len = int(max(0, min(50, pct / 2)))
|
| 28 |
+
bar = bar_char * bar_len
|
| 29 |
+
clean_name = name.split("\\")[-1].split("/")[-1][:50]
|
| 30 |
+
# lines.append(f" {clean_name:<50}: {bar:<50} {pct:5.1f}% (Var: {var:.6f})")
|
| 31 |
+
lines.append(f" {clean_name:<50}: {bar:<50} (Variance {pct:5.1f}%)")
|
| 32 |
+
|
| 33 |
+
log_entry = "\n".join(lines)
|
| 34 |
+
print(log_entry)
|
| 35 |
+
with open("sce_audit.log", "a", encoding="utf-8") as f:
|
| 36 |
+
f.write(log_entry + "\n")
|
| 37 |
+
|
| 38 |
+
|
| 39 |
+
@merge_method(
|
| 40 |
+
name="sce",
|
| 41 |
+
pretty_name="SCE",
|
| 42 |
+
reference_url="https://arxiv.org/abs/2408.07990",
|
| 43 |
+
)
|
| 44 |
+
def sce_merge(
|
| 45 |
+
tensors: List[torch.Tensor],
|
| 46 |
+
base_tensor: torch.Tensor,
|
| 47 |
+
output_weight: WeightInfo,
|
| 48 |
+
base_model: ModelReference,
|
| 49 |
+
int8_mask: bool = False,
|
| 50 |
+
select_topk: float = 1.0,
|
| 51 |
+
) -> torch.Tensor:
|
| 52 |
+
if not tensors:
|
| 53 |
+
return base_tensor
|
| 54 |
+
mask_dtype = torch.int8 if int8_mask else base_tensor.dtype
|
| 55 |
+
task_vectors = torch.stack([t - base_tensor for t in tensors], dim=0)
|
| 56 |
+
|
| 57 |
+
if select_topk < 1:
|
| 58 |
+
mask = sce_mask(task_vectors, select_topk, mask_dtype)
|
| 59 |
+
task_vectors = task_vectors * mask.unsqueeze(0)
|
| 60 |
+
|
| 61 |
+
# --- LIVE AUDIT CHART ---
|
| 62 |
+
model_refs = inspect.currentframe().f_back.f_locals.get('model_refs')
|
| 63 |
+
donor_names =[str(m.model.path) for m in model_refs] if model_refs else [f"Donor_{i}" for i in range(len(tensors))]
|
| 64 |
+
variances =[torch.var(tv.float()).item() for tv in task_vectors]
|
| 65 |
+
log_sce_audit(output_weight.name, str(base_model.model.path), donor_names, variances, select_topk)
|
| 66 |
+
# ------------------------
|
| 67 |
+
|
| 68 |
+
erase_mask = sign_consensus_mask(task_vectors, method="sum", mask_dtype=mask_dtype)
|
| 69 |
+
|
| 70 |
+
tv_weights = sce_weight(task_vectors)
|
| 71 |
+
while tv_weights.dim() < task_vectors.dim():
|
| 72 |
+
tv_weights = tv_weights.unsqueeze(-1)
|
| 73 |
+
|
| 74 |
+
erased_weights = tv_weights * erase_mask
|
| 75 |
+
merged_tv = (task_vectors * erased_weights).sum(dim=0)
|
| 76 |
+
final_tv = merged_tv / torch.sum(erased_weights, dim=0).clamp(min=1e-6)
|
| 77 |
+
|
| 78 |
+
return base_tensor + final_tv
|
| 79 |
+
|
| 80 |
+
|
| 81 |
+
def sce_weight(tvs: torch.Tensor) -> torch.Tensor:
|
| 82 |
+
weights = torch.mean(tvs**2, dim=list(range(1, tvs.dim())))
|
| 83 |
+
weight_sum = torch.sum(weights).item()
|
| 84 |
+
if abs(weight_sum) < 1e-6:
|
| 85 |
+
return torch.ones_like(weights) / weights.shape[0]
|
| 86 |
+
return weights / weight_sum
|
| 87 |
+
|
| 88 |
+
|
| 89 |
+
def sce_mask(
|
| 90 |
+
tvs: torch.Tensor, density: float, mask_dtype: Optional[torch.dtype] = None
|
| 91 |
+
):
|
| 92 |
+
if density <= 0:
|
| 93 |
+
return torch.zeros_like(tvs, dtype=mask_dtype)
|
| 94 |
+
if density >= 1:
|
| 95 |
+
return torch.ones_like(tvs, dtype=mask_dtype)
|
| 96 |
+
|
| 97 |
+
var = torch.var(tvs, dim=0, unbiased=False)
|
| 98 |
+
nonzero = torch.count_nonzero(var)
|
| 99 |
+
k = int(nonzero * density)
|
| 100 |
+
if k == 0:
|
| 101 |
+
return torch.zeros_like(tvs, dtype=mask_dtype)
|
| 102 |
+
|
| 103 |
+
_, indices = torch.topk(var.abs().view(-1), k=k, largest=True)
|
| 104 |
+
mask = torch.zeros_like(var, dtype=mask_dtype)
|
| 105 |
+
mask.view(-1)[indices] = 1
|
| 106 |
+
return mask
|