"""Accumulate per-expert input second moments (GPTQ Hessians) for a set of layers. For every routed (token, expert) pair of a layer visit: H13[e] += x x^T (input of gate_proj / up_proj, dim 2304) H2[e] += a a^T (input of down_proj, a = softcapped_swiglu(up_e x, gate_e x), dim 1152) Looped layers accumulate over all three visits. Usage: calib_capture.py """ import os, sys, time import numpy as np import mlx.core as mx sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) import qlab # noqa: E402 from mlx_vlm.models.inkling2 import language as L # noqa: E402 seq_path, out_dir, layer_arg = sys.argv[1:4] want = [int(x) for x in layer_arg.split(",")] os.makedirs(out_dir, exist_ok=True) model, _ = qlab.load_native() seqs = qlab.load_sequences(seq_path, split="calib", max_len=1024) state = {} for li in want: moe = qlab.layers(model)[li].mlp e, d, h = moe.gate_weight.shape[0], moe.gate_weight.shape[1], moe.switch_mlp.down_proj.input_dims state[id(moe)] = dict(layer=li, H13=[mx.zeros((d, d)) for _ in range(e)], H2=[mx.zeros((h, h)) for _ in range(e)], n=np.zeros(e, np.int64)) orig = L.SparseMoE.__call__ def patched(self, x): st = state.get(id(self)) if st is not None: indices, _ = L._route(x, self.gate_weight, self.e_score_correction_bias, self.top_k, self.route_scale) X = x.reshape(-1, x.shape[-1]) idx = np.array(indices.reshape(-1, indices.shape[-1])) flat = idx.reshape(-1) order = np.argsort(flat, kind="stable") tok_of = order // idx.shape[1] bounds = np.searchsorted(flat[order], np.arange(len(st["H13"]) + 1)) sw = self.switch_mlp H13, H2 = st["H13"], st["H2"] touched = [] for e in range(len(bounds) - 1): lo, hi = int(bounds[e]), int(bounds[e + 1]) if hi == lo: continue rows = mx.array(tok_of[lo:hi]) xe = X[rows] # (n, D), model dtype ei = mx.full((hi - lo, 1), e, dtype=mx.uint32) xin = xe[:, None, None, :] up = sw.up_proj(xin, ei) gate = sw.gate_proj(xin, ei) a = sw.activation(up, gate).reshape(hi - lo, -1).astype(mx.float32) xf = xe.astype(mx.float32) H13[e] = H13[e] + xf.T @ xf H2[e] = H2[e] + a.T @ a touched += [H13[e], H2[e]] st["n"][e] += hi - lo mx.eval(touched) return orig(self, x) L.SparseMoE.__call__ = patched t0 = time.time(); ntok = 0 for i, s in enumerate(seqs): out = model(mx.array([s["ids"]])) mx.eval(out.logits[0, -1]) ntok += len(s["ids"]) mx.clear_cache() if i % 10 == 0: print(f"seq {i}/{len(seqs)} tokens {ntok} elapsed {time.time() - t0:.0f}s peak {mx.get_peak_memory() / 2**30:.1f} GB", flush=True) for st in state.values(): li = st["layer"] mx.save_safetensors(os.path.join(out_dir, f"hess_L{li:02d}.safetensors"), {"H13": mx.stack(st["H13"]), "H2": mx.stack(st["H2"]), "n": mx.array(st["n"])}) print(f"layer {li}: routed pairs {int(st['n'].sum())} min/median/max per expert " f"{int(st['n'].min())}/{int(np.median(st['n']))}/{int(st['n'].max())}", flush=True) print("done", round(time.time() - t0), "s tokens", ntok)