Naphula commited on
Commit
3ceb796
·
verified ·
1 Parent(s): ae1ef82

sce audit was surprisingly difficult for some reason but it works now, rename to sce.py

Browse files
Files changed (1) hide show
  1. sce_audit.py +106 -0
sce_audit.py ADDED
@@ -0,0 +1,106 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Copyright (C) 2025 Arcee AI
2
+ # SPDX-License-Identifier: LGPL-3.0-only
3
+
4
+ from typing import List, Optional
5
+
6
+ import torch
7
+
8
+ from mergekit.merge_methods.easy_define import merge_method
9
+ from mergekit.merge_methods.generalized_task_arithmetic import (
10
+ get_mask as sign_consensus_mask,
11
+ )
12
+ from mergekit.architecture import WeightInfo
13
+ from mergekit.common import ModelReference
14
+ import inspect
15
+
16
+
17
+ def log_sce_audit(layer_name: str, base_name: str, donor_names: List[str], variances: List[float], select_topk: float):
18
+ bar_char = "█"
19
+ lines = [f"\n[SCE Audit] Layer: {layer_name} | select_topk={select_topk}"]
20
+ clean_base = base_name.split("\\")[-1].split("/")[-1][:50]
21
+ lines.append(f" [BASE] {clean_base:<50}")
22
+
23
+ total_var = sum(variances) if sum(variances) > 0 else 1.0
24
+
25
+ for name, var in zip(donor_names, variances):
26
+ pct = (var / total_var) * 100
27
+ bar_len = int(max(0, min(50, pct / 2)))
28
+ bar = bar_char * bar_len
29
+ clean_name = name.split("\\")[-1].split("/")[-1][:50]
30
+ # lines.append(f" {clean_name:<50}: {bar:<50} {pct:5.1f}% (Var: {var:.6f})")
31
+ lines.append(f" {clean_name:<50}: {bar:<50} (Variance {pct:5.1f}%)")
32
+
33
+ log_entry = "\n".join(lines)
34
+ print(log_entry)
35
+ with open("sce_audit.log", "a", encoding="utf-8") as f:
36
+ f.write(log_entry + "\n")
37
+
38
+
39
+ @merge_method(
40
+ name="sce",
41
+ pretty_name="SCE",
42
+ reference_url="https://arxiv.org/abs/2408.07990",
43
+ )
44
+ def sce_merge(
45
+ tensors: List[torch.Tensor],
46
+ base_tensor: torch.Tensor,
47
+ output_weight: WeightInfo,
48
+ base_model: ModelReference,
49
+ int8_mask: bool = False,
50
+ select_topk: float = 1.0,
51
+ ) -> torch.Tensor:
52
+ if not tensors:
53
+ return base_tensor
54
+ mask_dtype = torch.int8 if int8_mask else base_tensor.dtype
55
+ task_vectors = torch.stack([t - base_tensor for t in tensors], dim=0)
56
+
57
+ if select_topk < 1:
58
+ mask = sce_mask(task_vectors, select_topk, mask_dtype)
59
+ task_vectors = task_vectors * mask.unsqueeze(0)
60
+
61
+ # --- LIVE AUDIT CHART ---
62
+ model_refs = inspect.currentframe().f_back.f_locals.get('model_refs')
63
+ donor_names =[str(m.model.path) for m in model_refs] if model_refs else [f"Donor_{i}" for i in range(len(tensors))]
64
+ variances =[torch.var(tv.float()).item() for tv in task_vectors]
65
+ log_sce_audit(output_weight.name, str(base_model.model.path), donor_names, variances, select_topk)
66
+ # ------------------------
67
+
68
+ erase_mask = sign_consensus_mask(task_vectors, method="sum", mask_dtype=mask_dtype)
69
+
70
+ tv_weights = sce_weight(task_vectors)
71
+ while tv_weights.dim() < task_vectors.dim():
72
+ tv_weights = tv_weights.unsqueeze(-1)
73
+
74
+ erased_weights = tv_weights * erase_mask
75
+ merged_tv = (task_vectors * erased_weights).sum(dim=0)
76
+ final_tv = merged_tv / torch.sum(erased_weights, dim=0).clamp(min=1e-6)
77
+
78
+ return base_tensor + final_tv
79
+
80
+
81
+ def sce_weight(tvs: torch.Tensor) -> torch.Tensor:
82
+ weights = torch.mean(tvs**2, dim=list(range(1, tvs.dim())))
83
+ weight_sum = torch.sum(weights).item()
84
+ if abs(weight_sum) < 1e-6:
85
+ return torch.ones_like(weights) / weights.shape[0]
86
+ return weights / weight_sum
87
+
88
+
89
+ def sce_mask(
90
+ tvs: torch.Tensor, density: float, mask_dtype: Optional[torch.dtype] = None
91
+ ):
92
+ if density <= 0:
93
+ return torch.zeros_like(tvs, dtype=mask_dtype)
94
+ if density >= 1:
95
+ return torch.ones_like(tvs, dtype=mask_dtype)
96
+
97
+ var = torch.var(tvs, dim=0, unbiased=False)
98
+ nonzero = torch.count_nonzero(var)
99
+ k = int(nonzero * density)
100
+ if k == 0:
101
+ return torch.zeros_like(tvs, dtype=mask_dtype)
102
+
103
+ _, indices = torch.topk(var.abs().view(-1), k=k, largest=True)
104
+ mask = torch.zeros_like(var, dtype=mask_dtype)
105
+ mask.view(-1)[indices] = 1
106
+ return mask