File size: 3,401 Bytes
031489f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
"""Accumulate per-expert input second moments (GPTQ Hessians) for a set of layers.

For every routed (token, expert) pair of a layer visit:
  H13[e] += x x^T           (input of gate_proj / up_proj, dim 2304)
  H2[e]  += a a^T           (input of down_proj, a = softcapped_swiglu(up_e x, gate_e x), dim 1152)
Looped layers accumulate over all three visits. Usage: calib_capture.py <sequences.json> <out_dir> <layer,layer,...>
"""
import os, sys, time
import numpy as np
import mlx.core as mx

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import qlab  # noqa: E402
from mlx_vlm.models.inkling2 import language as L  # noqa: E402

seq_path, out_dir, layer_arg = sys.argv[1:4]
want = [int(x) for x in layer_arg.split(",")]
os.makedirs(out_dir, exist_ok=True)
model, _ = qlab.load_native()
seqs = qlab.load_sequences(seq_path, split="calib", max_len=1024)
state = {}
for li in want:
    moe = qlab.layers(model)[li].mlp
    e, d, h = moe.gate_weight.shape[0], moe.gate_weight.shape[1], moe.switch_mlp.down_proj.input_dims
    state[id(moe)] = dict(layer=li, H13=[mx.zeros((d, d)) for _ in range(e)], H2=[mx.zeros((h, h)) for _ in range(e)],
                          n=np.zeros(e, np.int64))
orig = L.SparseMoE.__call__

def patched(self, x):
    st = state.get(id(self))
    if st is not None:
        indices, _ = L._route(x, self.gate_weight, self.e_score_correction_bias, self.top_k, self.route_scale)
        X = x.reshape(-1, x.shape[-1])
        idx = np.array(indices.reshape(-1, indices.shape[-1]))
        flat = idx.reshape(-1)
        order = np.argsort(flat, kind="stable")
        tok_of = order // idx.shape[1]
        bounds = np.searchsorted(flat[order], np.arange(len(st["H13"]) + 1))
        sw = self.switch_mlp
        H13, H2 = st["H13"], st["H2"]
        touched = []
        for e in range(len(bounds) - 1):
            lo, hi = int(bounds[e]), int(bounds[e + 1])
            if hi == lo:
                continue
            rows = mx.array(tok_of[lo:hi])
            xe = X[rows]                                     # (n, D), model dtype
            ei = mx.full((hi - lo, 1), e, dtype=mx.uint32)
            xin = xe[:, None, None, :]
            up = sw.up_proj(xin, ei)
            gate = sw.gate_proj(xin, ei)
            a = sw.activation(up, gate).reshape(hi - lo, -1).astype(mx.float32)
            xf = xe.astype(mx.float32)
            H13[e] = H13[e] + xf.T @ xf
            H2[e] = H2[e] + a.T @ a
            touched += [H13[e], H2[e]]
            st["n"][e] += hi - lo
        mx.eval(touched)
    return orig(self, x)

L.SparseMoE.__call__ = patched
t0 = time.time(); ntok = 0
for i, s in enumerate(seqs):
    out = model(mx.array([s["ids"]]))
    mx.eval(out.logits[0, -1])
    ntok += len(s["ids"])
    mx.clear_cache()
    if i % 10 == 0:
        print(f"seq {i}/{len(seqs)} tokens {ntok} elapsed {time.time() - t0:.0f}s peak {mx.get_peak_memory() / 2**30:.1f} GB", flush=True)
for st in state.values():
    li = st["layer"]
    mx.save_safetensors(os.path.join(out_dir, f"hess_L{li:02d}.safetensors"),
                        {"H13": mx.stack(st["H13"]), "H2": mx.stack(st["H2"]), "n": mx.array(st["n"])})
    print(f"layer {li}: routed pairs {int(st['n'].sum())} min/median/max per expert "
          f"{int(st['n'].min())}/{int(np.median(st['n']))}/{int(st['n'].max())}", flush=True)
print("done", round(time.time() - t0), "s tokens", ntok)