"""Shared helpers for the Inkling 2 Mini quantization work (mlx-vlm layout).""" import json import math import os import mlx.core as mx import mlx.nn as nn import numpy as np from mlx.utils import tree_map from mlx_vlm import load from mlx_vlm.models.switch_layers import QuantizedSwitchLinear ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) NATIVE = os.path.join(ROOT, "upstream", "Inkling-2-Mini") PROJECTIONS = ("gate_proj", "up_proj", "down_proj") # ----------------------------------------------------------------------------- model access def load_native(fp32=False, path=NATIVE): model, processor = load(path) if fp32: to_fp32(model) mx.eval(model.parameters()) return model, processor def to_fp32(model): model.update(tree_map( lambda p: p.astype(mx.float32) if isinstance(p, mx.array) and p.dtype == mx.bfloat16 else p, model.parameters())) def layers(model): return model.language_model.model.layers def switch(model, layer): return layers(model)[layer].mlp.switch_mlp def dense_expert(proj): """Exact float32 weights (experts, out, in) of a native or affine expert projection.""" mode = proj.mode if mode == "nvfp4": w = mx.dequantize(proj.weight, proj.scales, None, group_size=16, bits=4, mode="nvfp4", dtype=mx.float32) scale = proj.global_scale.astype(mx.float32).reshape(-1, 1, 1) return w * scale if proj._scale_name == "weight_scale_2" else w / scale if mode == "mxfp8": return mx.dequantize(proj.weight, proj.scales, None, group_size=32, bits=8, mode="mxfp8", dtype=mx.float32) return mx.dequantize(proj.weight, proj.scales, proj.get("biases"), group_size=proj.group_size, bits=proj.bits, mode=mode, dtype=mx.float32) # ----------------------------------------------------------------------------- affine packing def pack_codes(codes, bits): """Pack integer codes (..., n) into MLX's uint32 layout: a little-endian bit stream along the last axis.""" n = codes.shape[-1] per = math.lcm(bits, 32) // bits # codes per repeating unit words = math.lcm(bits, 32) // 32 # uint32 words per unit assert n % per == 0, (n, per) c = codes.astype(mx.uint32).reshape(*codes.shape[:-1], n // per, per) out = [None] * words for j in range(per): off = bits * j k, sh = off // 32, off % 32 v = c[..., j] lo = mx.left_shift(v, mx.array(sh, mx.uint32)) if sh else v out[k] = lo if out[k] is None else mx.bitwise_or(out[k], lo) if sh + bits > 32: hi = mx.right_shift(v, mx.array(32 - sh, mx.uint32)) out[k + 1] = hi if out[k + 1] is None else mx.bitwise_or(out[k + 1], hi) return mx.stack(out, axis=-1).reshape(*codes.shape[:-1], n // per * words) def affine_switch(packed, scales, biases, bits, group_size): q = QuantizedSwitchLinear.__new__(QuantizedSwitchLinear) nn.Module.__init__(q) q.weight, q.scales, q.biases = packed, scales, biases q.group_size, q.bits, q.mode = group_size, bits, "affine" q.freeze() return q def rtn_affine(w, bits, group_size, scale_dtype=mx.float16): """MLX's own round-to-nearest affine quantizer, with scales stored in `scale_dtype`.""" wq, s, b = mx.quantize(w, group_size=group_size, bits=bits, mode="affine") return wq, s.astype(scale_dtype), b.astype(scale_dtype) # ----------------------------------------------------------------------------- data def render_ids(tok, rec): """Token ids of a recorded conversation (scenario record from gen_chats.py).""" import sys sys.path.insert(0, os.path.join(ROOT, "scripts")) from scenarios import TOOLS tools = [TOOLS[t] for t in rec.get("tools", [])] or None out = tok.apply_chat_template(rec["messages"], tools=tools, reasoning_effort=rec.get("effort"), tokenize=True) return list(out["input_ids"]) if hasattr(out, "keys") else list(out) def load_sequences(path, split=None, max_len=1024): """Token sequences from a prepared JSON file: [{"name", "split", "kind", "ids"}].""" seqs = json.load(open(path)) out = [] for s in seqs: if split and s["split"] != split: continue ids = s["ids"] for start in range(0, len(ids), max_len): chunk = ids[start:start + max_len] if len(chunk) >= 16: out.append(dict(name=f"{s['name']}@{start}", kind=s["kind"], ids=chunk)) return out def logits_for(model, ids): out = model(mx.array([ids])).logits[0].astype(mx.float32) mx.eval(out) return out def log_softmax(x): return x - mx.logsumexp(x, axis=-1, keepdims=True) def compare(teacher_logits, student_logits, ids): """Per-sequence statistics; teacher and student are (T, V) float32 logits.""" lt, ls = log_softmax(teacher_logits), log_softmax(student_logits) kl = (mx.exp(lt) * (lt - ls)).sum(-1) nxt = mx.array(ids[1:]) nll_t = -mx.take_along_axis(lt[:-1], nxt[:, None], axis=-1)[:, 0] nll_s = -mx.take_along_axis(ls[:-1], nxt[:, None], axis=-1)[:, 0] agree = (mx.argmax(lt, -1) == mx.argmax(ls, -1)).astype(mx.float32) vals = [kl.sum(), kl.max(), agree.sum(), nll_t.sum(), nll_s.sum()] mx.eval(vals) return dict(n=len(ids), kl_sum=vals[0].item(), kl_max=vals[1].item(), agree=vals[2].item(), nll_t=vals[3].item(), nll_s=vals[4].item()) def summarize(rows): n = sum(r["n"] for r in rows) m = sum(r["n"] - 1 for r in rows) return dict(tokens=n, kl=sum(r["kl_sum"] for r in rows) / n, kl_max=max(r["kl_max"] for r in rows), top1=sum(r["agree"] for r in rows) / n, nll_teacher=sum(r["nll_t"] for r in rows) / m, nll_student=sum(r["nll_s"] for r in rows) / m)