VisualInference's picture
Inkling 2 Mini for MLX, 3bit: weights, card, validation results
e0b70af verified
Raw History Blame Contribute Delete
5.8 kB
"""Shared helpers for the Inkling 2 Mini quantization work (mlx-vlm layout)."""
import json
import math
import os
import mlx.core as mx
import mlx.nn as nn
import numpy as np
from mlx.utils import tree_map
from mlx_vlm import load
from mlx_vlm.models.switch_layers import QuantizedSwitchLinear
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
NATIVE = os.path.join(ROOT, "upstream", "Inkling-2-Mini")
PROJECTIONS = ("gate_proj", "up_proj", "down_proj")
# ----------------------------------------------------------------------------- model access
def load_native(fp32=False, path=NATIVE):
model, processor = load(path)
if fp32:
to_fp32(model)
mx.eval(model.parameters())
return model, processor
def to_fp32(model):
model.update(tree_map(
lambda p: p.astype(mx.float32) if isinstance(p, mx.array) and p.dtype == mx.bfloat16 else p,
model.parameters()))
def layers(model):
return model.language_model.model.layers
def switch(model, layer):
return layers(model)[layer].mlp.switch_mlp
def dense_expert(proj):
"""Exact float32 weights (experts, out, in) of a native or affine expert projection."""
mode = proj.mode
if mode == "nvfp4":
w = mx.dequantize(proj.weight, proj.scales, None, group_size=16, bits=4, mode="nvfp4", dtype=mx.float32)
scale = proj.global_scale.astype(mx.float32).reshape(-1, 1, 1)
return w * scale if proj._scale_name == "weight_scale_2" else w / scale
if mode == "mxfp8":
return mx.dequantize(proj.weight, proj.scales, None, group_size=32, bits=8, mode="mxfp8", dtype=mx.float32)
return mx.dequantize(proj.weight, proj.scales, proj.get("biases"), group_size=proj.group_size, bits=proj.bits,
mode=mode, dtype=mx.float32)
# ----------------------------------------------------------------------------- affine packing
def pack_codes(codes, bits):
"""Pack integer codes (..., n) into MLX's uint32 layout: a little-endian bit stream along the last axis."""
n = codes.shape[-1]
per = math.lcm(bits, 32) // bits # codes per repeating unit
words = math.lcm(bits, 32) // 32 # uint32 words per unit
assert n % per == 0, (n, per)
c = codes.astype(mx.uint32).reshape(*codes.shape[:-1], n // per, per)
out = [None] * words
for j in range(per):
off = bits * j
k, sh = off // 32, off % 32
v = c[..., j]
lo = mx.left_shift(v, mx.array(sh, mx.uint32)) if sh else v
out[k] = lo if out[k] is None else mx.bitwise_or(out[k], lo)
if sh + bits > 32:
hi = mx.right_shift(v, mx.array(32 - sh, mx.uint32))
out[k + 1] = hi if out[k + 1] is None else mx.bitwise_or(out[k + 1], hi)
return mx.stack(out, axis=-1).reshape(*codes.shape[:-1], n // per * words)
def affine_switch(packed, scales, biases, bits, group_size):
q = QuantizedSwitchLinear.__new__(QuantizedSwitchLinear)
nn.Module.__init__(q)
q.weight, q.scales, q.biases = packed, scales, biases
q.group_size, q.bits, q.mode = group_size, bits, "affine"
q.freeze()
return q
def rtn_affine(w, bits, group_size, scale_dtype=mx.float16):
"""MLX's own round-to-nearest affine quantizer, with scales stored in `scale_dtype`."""
wq, s, b = mx.quantize(w, group_size=group_size, bits=bits, mode="affine")
return wq, s.astype(scale_dtype), b.astype(scale_dtype)
# ----------------------------------------------------------------------------- data
def render_ids(tok, rec):
"""Token ids of a recorded conversation (scenario record from gen_chats.py)."""
import sys
sys.path.insert(0, os.path.join(ROOT, "scripts"))
from scenarios import TOOLS
tools = [TOOLS[t] for t in rec.get("tools", [])] or None
out = tok.apply_chat_template(rec["messages"], tools=tools, reasoning_effort=rec.get("effort"), tokenize=True)
return list(out["input_ids"]) if hasattr(out, "keys") else list(out)
def load_sequences(path, split=None, max_len=1024):
"""Token sequences from a prepared JSON file: [{"name", "split", "kind", "ids"}]."""
seqs = json.load(open(path))
out = []
for s in seqs:
if split and s["split"] != split:
continue
ids = s["ids"]
for start in range(0, len(ids), max_len):
chunk = ids[start:start + max_len]
if len(chunk) >= 16:
out.append(dict(name=f"{s['name']}@{start}", kind=s["kind"], ids=chunk))
return out
def logits_for(model, ids):
out = model(mx.array([ids])).logits[0].astype(mx.float32)
mx.eval(out)
return out
def log_softmax(x):
return x - mx.logsumexp(x, axis=-1, keepdims=True)
def compare(teacher_logits, student_logits, ids):
"""Per-sequence statistics; teacher and student are (T, V) float32 logits."""
lt, ls = log_softmax(teacher_logits), log_softmax(student_logits)
kl = (mx.exp(lt) * (lt - ls)).sum(-1)
nxt = mx.array(ids[1:])
nll_t = -mx.take_along_axis(lt[:-1], nxt[:, None], axis=-1)[:, 0]
nll_s = -mx.take_along_axis(ls[:-1], nxt[:, None], axis=-1)[:, 0]
agree = (mx.argmax(lt, -1) == mx.argmax(ls, -1)).astype(mx.float32)
vals = [kl.sum(), kl.max(), agree.sum(), nll_t.sum(), nll_s.sum()]
mx.eval(vals)
return dict(n=len(ids), kl_sum=vals[0].item(), kl_max=vals[1].item(), agree=vals[2].item(),
nll_t=vals[3].item(), nll_s=vals[4].item())
def summarize(rows):
n = sum(r["n"] for r in rows)
m = sum(r["n"] - 1 for r in rows)
return dict(tokens=n, kl=sum(r["kl_sum"] for r in rows) / n, kl_max=max(r["kl_max"] for r in rows),
top1=sum(r["agree"] for r in rows) / n, nll_teacher=sum(r["nll_t"] for r in rows) / m,
nll_student=sum(r["nll_s"] for r in rows) / m)