"""Apply a quantization recipe to the loaded native model (in memory), and describe it for config.json. A recipe is a dict: {"experts": {"default": SPEC, "layers": {"20": SPEC, ...}}, SPEC = "native" | "rtn:BITS:GROUP" | "gptq:BITS:GROUP" "dense": {"attn": Q, "embed": Q, "head": Q, "vision": Q, "draft": Q}} Q = null (keep float) | [BITS, GROUP] "gptq" shards are read from `qdir` (files q_LNN_bBgG.safetensors written by gptq.py). """ import os import mlx.core as mx import mlx.nn as nn import qlab ATTN_BIG = ("q_proj", "k_proj", "v_proj", "r_proj", "o_proj", "gate_proj") def expert_spec(recipe, layer): ex = recipe.get("experts", {}) return ex.get("layers", {}).get(str(layer), ex.get("default", "native")) def dense_group(path, module): """Which recipe group a dense module belongs to, or None if it must stay in floating point.""" if ".mlp.switch_mlp." in path: return None if path == "language_model.model.embed_tokens": return "embed" if path == "language_model.lm_head": return "head" if path.startswith("language_model.model.layers.") and ".self_attn." in path: leaf = path.rsplit(".", 1)[-1] # sink_proj (24 rows) and the dynamic-convolution tap projections are tiny and stay bf16. return "attn" if leaf in ATTN_BIG and ".k_sconv." not in path and ".v_sconv." not in path else None if path.startswith("vision_tower."): return "vision" return None def apply_experts(model, recipe, qdir=None, log=None): """Replace expert projections according to the recipe. Returns {module_path: quantization dict}.""" overrides = {} act_dtype = model.language_model.model.embed_norm.weight.dtype for li, layer in enumerate(qlab.layers(model)): spec = expert_spec(recipe, li) sw = layer.mlp.switch_mlp base = f"language_model.model.layers.{li}.mlp.switch_mlp." if spec == "native": for p in qlab.PROJECTIONS: proj = getattr(sw, p) overrides[base + p] = {"group_size": proj.group_size, "bits": proj.bits, "mode": proj.mode} continue kind, bits, group = spec.split(":") bits, group = int(bits), int(group) if kind == "gptq": shard = mx.load(os.path.join(qdir, f"q_L{li:02d}_b{bits}g{group}.safetensors")) for p in qlab.PROJECTIONS: if kind == "gptq": packed, s, b = shard[p + ".weight"], shard[p + ".scales"], shard[p + ".biases"] elif kind == "rtn": packed, s, b = qlab.rtn_affine(qlab.dense_expert(getattr(sw, p)), bits, group) else: raise ValueError(spec) # Scales and offsets are stored in the activation dtype. MLX's expert kernel (gather_qmm) promotes to # float32 and runs about three times slower when their dtype differs from the input's. s, b = s.astype(act_dtype), b.astype(act_dtype) mx.eval(packed, s, b) setattr(sw, p, qlab.affine_switch(packed, s, b, bits, group)) overrides[base + p] = {"group_size": group, "bits": bits, "mode": "affine"} mx.clear_cache() if log: log(f"layer {li}: experts -> {spec}") return overrides def apply_dense(model, recipe): """Affine-quantize dense groups. Returns {module_path: quantization dict}.""" dense = recipe.get("dense", {}) or {} overrides = {} def predicate(path, module): if not hasattr(module, "to_quantized") or not hasattr(module, "weight"): return False q = dense.get(dense_group(path, module)) if not q: return False bits, group = q if module.weight.shape[-1] % group != 0: return False overrides[path] = {"group_size": group, "bits": bits, "mode": "affine"} return {"group_size": group, "bits": bits, "mode": "affine"} nn.quantize(model, class_predicate=predicate) return overrides def apply(model, recipe, qdir=None, log=None): overrides = apply_experts(model, recipe, qdir, log) overrides.update(apply_dense(model, recipe)) mx.eval(model.parameters()) return overrides def nbytes(model): from mlx.utils import tree_flatten return sum(v.nbytes for _, v in tree_flatten(model.parameters()))