File size: 4,125 Bytes
031489f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
"""Emulated activation quantization for the expert projections (experiment).

hf_quant_config.json lists `required_activation_quantization` for the expert weights:
  layers 0-19  nvfp4_e2m1, block 16, per-token amax, E4M3 block scales, scale rule "fouroversix_mae"
  layers 20-23 mxfp8_e4m3, block 32, power-of-two (E8M0) block scales
Neither transformers nor the MLX runtimes apply it. These functions quantize-and-dequantize the inputs of
gate/up (the normed hidden state) and of down (the gated activation) so the effect on held-out loss can be measured.
The exact rounding conventions of the original runtime are not published; this follows the public formats.
"""

import mlx.core as mx

E4M3_MAX = 448.0
E4M3_MIN = 2.0 ** -6


def round_e4m3(v):
    """Round positive values to the nearest E4M3 normal number (3 mantissa bits), clamped to the format's range."""
    v = mx.clip(v, E4M3_MIN, E4M3_MAX)
    e = mx.floor(mx.log2(v))
    m = mx.round((v / mx.power(2.0, e) - 1.0) * 8.0)
    return mx.minimum(mx.power(2.0, e) * (1.0 + m / 8.0), E4M3_MAX)


def round_e4m3_signed(v):
    a = mx.abs(v)
    q = mx.where(a < E4M3_MIN / 2, 0.0, round_e4m3(a))
    return mx.sign(v) * q


def round_e2m1(v):
    """Nearest value of {0, .5, 1, 1.5, 2, 3, 4, 6} with sign."""
    a = mx.minimum(mx.abs(v), 6.0)
    q = mx.where(a < 2.0, mx.round(a * 2.0) / 2.0, mx.where(a < 4.0, mx.round(a), mx.where(a < 5.0, 4.0, 6.0)))
    return mx.sign(v) * q


def nvfp4_fake_quant(x):
    dtype = x.dtype
    x = x.astype(mx.float32)
    d = x.shape[-1]
    amax = mx.abs(x).max(axis=-1, keepdims=True)                     # per token
    outer = mx.maximum(amax, 1e-12) / (6.0 * E4M3_MAX)
    b = x.reshape(*x.shape[:-1], d // 16, 16)
    bmax = mx.maximum(mx.abs(b).max(axis=-1, keepdims=True), 1e-12)
    out = err = None
    for top in (6.0, 4.0):                                           # "four over six": map the block max to 6 or to 4
        scale = round_e4m3(bmax / top / outer[..., None]) * outer[..., None]
        q = round_e2m1(b / scale) * scale
        e = mx.abs(q - b).mean(axis=-1, keepdims=True)
        if out is None:
            out, err = q, e
        else:
            out = mx.where(e < err, q, out)
    return out.reshape(x.shape).astype(dtype)


def mxfp8_fake_quant(x):
    dtype = x.dtype
    x = x.astype(mx.float32)
    d = x.shape[-1]
    b = x.reshape(*x.shape[:-1], d // 32, 32)
    bmax = mx.maximum(mx.abs(b).max(axis=-1, keepdims=True), 1e-30)
    scale = mx.power(2.0, mx.ceil(mx.log2(bmax / E4M3_MAX)))
    return (round_e4m3_signed(b / scale) * scale).reshape(x.shape).astype(dtype)


def install(model, which=("input", "hidden")):
    """Patch every layer's MoE to quantize expert inputs. Returns an undo function."""
    from mlx_vlm.models.inkling2 import language as L

    originals = []
    for layer in model.language_model.model.layers:
        moe = layer.mlp
        fq = nvfp4_fake_quant if getattr(moe.switch_mlp.down_proj, "mode", "") != "mxfp8" else mxfp8_fake_quant
        moe._fq = fq
        act = moe.switch_mlp.activation
        originals.append((moe, act))
        if "hidden" in which:
            class QuantAct(L.SoftcappedSwiGLU):
                def __call__(self, up, gate, _fq=fq, _base=act):
                    return _fq(_base(up, gate))
            new = QuantAct.__new__(QuantAct)
            import mlx.nn as nn
            nn.Module.__init__(new)
            new.gate_cap, new.up_cap = act.gate_cap, act.up_cap
            moe.switch_mlp.activation = new
    orig_call = L.SparseMoE.__call__

    def patched(self, x):
        if "input" not in which or getattr(self, "_fq", None) is None:
            return orig_call(self, x)
        indices, weights = L._route(x, self.gate_weight, self.e_score_correction_bias, self.top_k, self.route_scale)
        out = self.switch_mlp(self._fq(x), indices)
        return L._combine_routes(out, weights)

    L.SparseMoE.__call__ = patched

    def undo():
        L.SparseMoE.__call__ = orig_call
        for moe, act in originals:
            moe.switch_mlp.activation = act
            moe._fq = None
    return undo