VisualInference's picture
Inkling 2 Mini for MLX, 4bit: weights, card, validation results
d19188c verified
Raw History Blame Contribute Delete
4.35 kB
"""Apply a quantization recipe to the loaded native model (in memory), and describe it for config.json.
A recipe is a dict:
{"experts": {"default": SPEC, "layers": {"20": SPEC, ...}}, SPEC = "native" | "rtn:BITS:GROUP" | "gptq:BITS:GROUP"
"dense": {"attn": Q, "embed": Q, "head": Q, "vision": Q, "draft": Q}} Q = null (keep float) | [BITS, GROUP]
"gptq" shards are read from `qdir` (files q_LNN_bBgG.safetensors written by gptq.py).
"""
import os
import mlx.core as mx
import mlx.nn as nn
import qlab
ATTN_BIG = ("q_proj", "k_proj", "v_proj", "r_proj", "o_proj", "gate_proj")
def expert_spec(recipe, layer):
ex = recipe.get("experts", {})
return ex.get("layers", {}).get(str(layer), ex.get("default", "native"))
def dense_group(path, module):
"""Which recipe group a dense module belongs to, or None if it must stay in floating point."""
if ".mlp.switch_mlp." in path:
return None
if path == "language_model.model.embed_tokens":
return "embed"
if path == "language_model.lm_head":
return "head"
if path.startswith("language_model.model.layers.") and ".self_attn." in path:
leaf = path.rsplit(".", 1)[-1]
# sink_proj (24 rows) and the dynamic-convolution tap projections are tiny and stay bf16.
return "attn" if leaf in ATTN_BIG and ".k_sconv." not in path and ".v_sconv." not in path else None
if path.startswith("vision_tower."):
return "vision"
return None
def apply_experts(model, recipe, qdir=None, log=None):
"""Replace expert projections according to the recipe. Returns {module_path: quantization dict}."""
overrides = {}
act_dtype = model.language_model.model.embed_norm.weight.dtype
for li, layer in enumerate(qlab.layers(model)):
spec = expert_spec(recipe, li)
sw = layer.mlp.switch_mlp
base = f"language_model.model.layers.{li}.mlp.switch_mlp."
if spec == "native":
for p in qlab.PROJECTIONS:
proj = getattr(sw, p)
overrides[base + p] = {"group_size": proj.group_size, "bits": proj.bits, "mode": proj.mode}
continue
kind, bits, group = spec.split(":")
bits, group = int(bits), int(group)
if kind == "gptq":
shard = mx.load(os.path.join(qdir, f"q_L{li:02d}_b{bits}g{group}.safetensors"))
for p in qlab.PROJECTIONS:
if kind == "gptq":
packed, s, b = shard[p + ".weight"], shard[p + ".scales"], shard[p + ".biases"]
elif kind == "rtn":
packed, s, b = qlab.rtn_affine(qlab.dense_expert(getattr(sw, p)), bits, group)
else:
raise ValueError(spec)
# Scales and offsets are stored in the activation dtype. MLX's expert kernel (gather_qmm) promotes to
# float32 and runs about three times slower when their dtype differs from the input's.
s, b = s.astype(act_dtype), b.astype(act_dtype)
mx.eval(packed, s, b)
setattr(sw, p, qlab.affine_switch(packed, s, b, bits, group))
overrides[base + p] = {"group_size": group, "bits": bits, "mode": "affine"}
mx.clear_cache()
if log:
log(f"layer {li}: experts -> {spec}")
return overrides
def apply_dense(model, recipe):
"""Affine-quantize dense groups. Returns {module_path: quantization dict}."""
dense = recipe.get("dense", {}) or {}
overrides = {}
def predicate(path, module):
if not hasattr(module, "to_quantized") or not hasattr(module, "weight"):
return False
q = dense.get(dense_group(path, module))
if not q:
return False
bits, group = q
if module.weight.shape[-1] % group != 0:
return False
overrides[path] = {"group_size": group, "bits": bits, "mode": "affine"}
return {"group_size": group, "bits": bits, "mode": "affine"}
nn.quantize(model, class_predicate=predicate)
return overrides
def apply(model, recipe, qdir=None, log=None):
overrides = apply_experts(model, recipe, qdir, log)
overrides.update(apply_dense(model, recipe))
mx.eval(model.parameters())
return overrides
def nbytes(model):
from mlx.utils import tree_flatten
return sum(v.nbytes for _, v in tree_flatten(model.parameters()))