File size: 4,349 Bytes
e0b70af
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
"""Apply a quantization recipe to the loaded native model (in memory), and describe it for config.json.

A recipe is a dict:
  {"experts": {"default": SPEC, "layers": {"20": SPEC, ...}},        SPEC = "native" | "rtn:BITS:GROUP" | "gptq:BITS:GROUP"
   "dense":   {"attn": Q, "embed": Q, "head": Q, "vision": Q, "draft": Q}}   Q = null (keep float) | [BITS, GROUP]
"gptq" shards are read from `qdir` (files q_LNN_bBgG.safetensors written by gptq.py).
"""

import os

import mlx.core as mx
import mlx.nn as nn

import qlab

ATTN_BIG = ("q_proj", "k_proj", "v_proj", "r_proj", "o_proj", "gate_proj")


def expert_spec(recipe, layer):
    ex = recipe.get("experts", {})
    return ex.get("layers", {}).get(str(layer), ex.get("default", "native"))


def dense_group(path, module):
    """Which recipe group a dense module belongs to, or None if it must stay in floating point."""
    if ".mlp.switch_mlp." in path:
        return None
    if path == "language_model.model.embed_tokens":
        return "embed"
    if path == "language_model.lm_head":
        return "head"
    if path.startswith("language_model.model.layers.") and ".self_attn." in path:
        leaf = path.rsplit(".", 1)[-1]
        # sink_proj (24 rows) and the dynamic-convolution tap projections are tiny and stay bf16.
        return "attn" if leaf in ATTN_BIG and ".k_sconv." not in path and ".v_sconv." not in path else None
    if path.startswith("vision_tower."):
        return "vision"
    return None


def apply_experts(model, recipe, qdir=None, log=None):
    """Replace expert projections according to the recipe. Returns {module_path: quantization dict}."""
    overrides = {}
    act_dtype = model.language_model.model.embed_norm.weight.dtype
    for li, layer in enumerate(qlab.layers(model)):
        spec = expert_spec(recipe, li)
        sw = layer.mlp.switch_mlp
        base = f"language_model.model.layers.{li}.mlp.switch_mlp."
        if spec == "native":
            for p in qlab.PROJECTIONS:
                proj = getattr(sw, p)
                overrides[base + p] = {"group_size": proj.group_size, "bits": proj.bits, "mode": proj.mode}
            continue
        kind, bits, group = spec.split(":")
        bits, group = int(bits), int(group)
        if kind == "gptq":
            shard = mx.load(os.path.join(qdir, f"q_L{li:02d}_b{bits}g{group}.safetensors"))
        for p in qlab.PROJECTIONS:
            if kind == "gptq":
                packed, s, b = shard[p + ".weight"], shard[p + ".scales"], shard[p + ".biases"]
            elif kind == "rtn":
                packed, s, b = qlab.rtn_affine(qlab.dense_expert(getattr(sw, p)), bits, group)
            else:
                raise ValueError(spec)
            # Scales and offsets are stored in the activation dtype. MLX's expert kernel (gather_qmm) promotes to
            # float32 and runs about three times slower when their dtype differs from the input's.
            s, b = s.astype(act_dtype), b.astype(act_dtype)
            mx.eval(packed, s, b)
            setattr(sw, p, qlab.affine_switch(packed, s, b, bits, group))
            overrides[base + p] = {"group_size": group, "bits": bits, "mode": "affine"}
        mx.clear_cache()
        if log:
            log(f"layer {li}: experts -> {spec}")
    return overrides


def apply_dense(model, recipe):
    """Affine-quantize dense groups. Returns {module_path: quantization dict}."""
    dense = recipe.get("dense", {}) or {}
    overrides = {}

    def predicate(path, module):
        if not hasattr(module, "to_quantized") or not hasattr(module, "weight"):
            return False
        q = dense.get(dense_group(path, module))
        if not q:
            return False
        bits, group = q
        if module.weight.shape[-1] % group != 0:
            return False
        overrides[path] = {"group_size": group, "bits": bits, "mode": "affine"}
        return {"group_size": group, "bits": bits, "mode": "affine"}

    nn.quantize(model, class_predicate=predicate)
    return overrides


def apply(model, recipe, qdir=None, log=None):
    overrides = apply_experts(model, recipe, qdir, log)
    overrides.update(apply_dense(model, recipe))
    mx.eval(model.parameters())
    return overrides


def nbytes(model):
    from mlx.utils import tree_flatten
    return sum(v.nbytes for _, v in tree_flatten(model.parameters()))