"""Emulated activation quantization for the expert projections (experiment). hf_quant_config.json lists `required_activation_quantization` for the expert weights: layers 0-19 nvfp4_e2m1, block 16, per-token amax, E4M3 block scales, scale rule "fouroversix_mae" layers 20-23 mxfp8_e4m3, block 32, power-of-two (E8M0) block scales Neither transformers nor the MLX runtimes apply it. These functions quantize-and-dequantize the inputs of gate/up (the normed hidden state) and of down (the gated activation) so the effect on held-out loss can be measured. The exact rounding conventions of the original runtime are not published; this follows the public formats. """ import mlx.core as mx E4M3_MAX = 448.0 E4M3_MIN = 2.0 ** -6 def round_e4m3(v): """Round positive values to the nearest E4M3 normal number (3 mantissa bits), clamped to the format's range.""" v = mx.clip(v, E4M3_MIN, E4M3_MAX) e = mx.floor(mx.log2(v)) m = mx.round((v / mx.power(2.0, e) - 1.0) * 8.0) return mx.minimum(mx.power(2.0, e) * (1.0 + m / 8.0), E4M3_MAX) def round_e4m3_signed(v): a = mx.abs(v) q = mx.where(a < E4M3_MIN / 2, 0.0, round_e4m3(a)) return mx.sign(v) * q def round_e2m1(v): """Nearest value of {0, .5, 1, 1.5, 2, 3, 4, 6} with sign.""" a = mx.minimum(mx.abs(v), 6.0) q = mx.where(a < 2.0, mx.round(a * 2.0) / 2.0, mx.where(a < 4.0, mx.round(a), mx.where(a < 5.0, 4.0, 6.0))) return mx.sign(v) * q def nvfp4_fake_quant(x): dtype = x.dtype x = x.astype(mx.float32) d = x.shape[-1] amax = mx.abs(x).max(axis=-1, keepdims=True) # per token outer = mx.maximum(amax, 1e-12) / (6.0 * E4M3_MAX) b = x.reshape(*x.shape[:-1], d // 16, 16) bmax = mx.maximum(mx.abs(b).max(axis=-1, keepdims=True), 1e-12) out = err = None for top in (6.0, 4.0): # "four over six": map the block max to 6 or to 4 scale = round_e4m3(bmax / top / outer[..., None]) * outer[..., None] q = round_e2m1(b / scale) * scale e = mx.abs(q - b).mean(axis=-1, keepdims=True) if out is None: out, err = q, e else: out = mx.where(e < err, q, out) return out.reshape(x.shape).astype(dtype) def mxfp8_fake_quant(x): dtype = x.dtype x = x.astype(mx.float32) d = x.shape[-1] b = x.reshape(*x.shape[:-1], d // 32, 32) bmax = mx.maximum(mx.abs(b).max(axis=-1, keepdims=True), 1e-30) scale = mx.power(2.0, mx.ceil(mx.log2(bmax / E4M3_MAX))) return (round_e4m3_signed(b / scale) * scale).reshape(x.shape).astype(dtype) def install(model, which=("input", "hidden")): """Patch every layer's MoE to quantize expert inputs. Returns an undo function.""" from mlx_vlm.models.inkling2 import language as L originals = [] for layer in model.language_model.model.layers: moe = layer.mlp fq = nvfp4_fake_quant if getattr(moe.switch_mlp.down_proj, "mode", "") != "mxfp8" else mxfp8_fake_quant moe._fq = fq act = moe.switch_mlp.activation originals.append((moe, act)) if "hidden" in which: class QuantAct(L.SoftcappedSwiGLU): def __call__(self, up, gate, _fq=fq, _base=act): return _fq(_base(up, gate)) new = QuantAct.__new__(QuantAct) import mlx.nn as nn nn.Module.__init__(new) new.gate_cap, new.up_cap = act.gate_cap, act.up_cap moe.switch_mlp.activation = new orig_call = L.SparseMoE.__call__ def patched(self, x): if "input" not in which or getattr(self, "_fq", None) is None: return orig_call(self, x) indices, weights = L._route(x, self.gate_weight, self.e_score_correction_bias, self.top_k, self.route_scale) out = self.switch_mlp(self._fq(x), indices) return L._combine_routes(out, weights) L.SparseMoE.__call__ = patched def undo(): L.SparseMoE.__call__ = orig_call for moe, act in originals: moe.switch_mlp.activation = act moe._fq = None return undo