VisualInference's picture
Inkling 2 Mini for MLX, 2bit: weights, card, validation results
031489f verified
Raw History Blame Contribute Delete
4.13 kB
"""Emulated activation quantization for the expert projections (experiment).
hf_quant_config.json lists `required_activation_quantization` for the expert weights:
layers 0-19 nvfp4_e2m1, block 16, per-token amax, E4M3 block scales, scale rule "fouroversix_mae"
layers 20-23 mxfp8_e4m3, block 32, power-of-two (E8M0) block scales
Neither transformers nor the MLX runtimes apply it. These functions quantize-and-dequantize the inputs of
gate/up (the normed hidden state) and of down (the gated activation) so the effect on held-out loss can be measured.
The exact rounding conventions of the original runtime are not published; this follows the public formats.
"""
import mlx.core as mx
E4M3_MAX = 448.0
E4M3_MIN = 2.0 ** -6
def round_e4m3(v):
"""Round positive values to the nearest E4M3 normal number (3 mantissa bits), clamped to the format's range."""
v = mx.clip(v, E4M3_MIN, E4M3_MAX)
e = mx.floor(mx.log2(v))
m = mx.round((v / mx.power(2.0, e) - 1.0) * 8.0)
return mx.minimum(mx.power(2.0, e) * (1.0 + m / 8.0), E4M3_MAX)
def round_e4m3_signed(v):
a = mx.abs(v)
q = mx.where(a < E4M3_MIN / 2, 0.0, round_e4m3(a))
return mx.sign(v) * q
def round_e2m1(v):
"""Nearest value of {0, .5, 1, 1.5, 2, 3, 4, 6} with sign."""
a = mx.minimum(mx.abs(v), 6.0)
q = mx.where(a < 2.0, mx.round(a * 2.0) / 2.0, mx.where(a < 4.0, mx.round(a), mx.where(a < 5.0, 4.0, 6.0)))
return mx.sign(v) * q
def nvfp4_fake_quant(x):
dtype = x.dtype
x = x.astype(mx.float32)
d = x.shape[-1]
amax = mx.abs(x).max(axis=-1, keepdims=True) # per token
outer = mx.maximum(amax, 1e-12) / (6.0 * E4M3_MAX)
b = x.reshape(*x.shape[:-1], d // 16, 16)
bmax = mx.maximum(mx.abs(b).max(axis=-1, keepdims=True), 1e-12)
out = err = None
for top in (6.0, 4.0): # "four over six": map the block max to 6 or to 4
scale = round_e4m3(bmax / top / outer[..., None]) * outer[..., None]
q = round_e2m1(b / scale) * scale
e = mx.abs(q - b).mean(axis=-1, keepdims=True)
if out is None:
out, err = q, e
else:
out = mx.where(e < err, q, out)
return out.reshape(x.shape).astype(dtype)
def mxfp8_fake_quant(x):
dtype = x.dtype
x = x.astype(mx.float32)
d = x.shape[-1]
b = x.reshape(*x.shape[:-1], d // 32, 32)
bmax = mx.maximum(mx.abs(b).max(axis=-1, keepdims=True), 1e-30)
scale = mx.power(2.0, mx.ceil(mx.log2(bmax / E4M3_MAX)))
return (round_e4m3_signed(b / scale) * scale).reshape(x.shape).astype(dtype)
def install(model, which=("input", "hidden")):
"""Patch every layer's MoE to quantize expert inputs. Returns an undo function."""
from mlx_vlm.models.inkling2 import language as L
originals = []
for layer in model.language_model.model.layers:
moe = layer.mlp
fq = nvfp4_fake_quant if getattr(moe.switch_mlp.down_proj, "mode", "") != "mxfp8" else mxfp8_fake_quant
moe._fq = fq
act = moe.switch_mlp.activation
originals.append((moe, act))
if "hidden" in which:
class QuantAct(L.SoftcappedSwiGLU):
def __call__(self, up, gate, _fq=fq, _base=act):
return _fq(_base(up, gate))
new = QuantAct.__new__(QuantAct)
import mlx.nn as nn
nn.Module.__init__(new)
new.gate_cap, new.up_cap = act.gate_cap, act.up_cap
moe.switch_mlp.activation = new
orig_call = L.SparseMoE.__call__
def patched(self, x):
if "input" not in which or getattr(self, "_fq", None) is None:
return orig_call(self, x)
indices, weights = L._route(x, self.gate_weight, self.e_score_correction_bias, self.top_k, self.route_scale)
out = self.switch_mlp(self._fq(x), indices)
return L._combine_routes(out, weights)
L.SparseMoE.__call__ = patched
def undo():
L.SparseMoE.__call__ = orig_call
for moe, act in originals:
moe.switch_mlp.activation = act
moe._fq = None
return undo