File size: 5,799 Bytes
d19188c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
"""Shared helpers for the Inkling 2 Mini quantization work (mlx-vlm layout)."""

import json
import math
import os

import mlx.core as mx
import mlx.nn as nn
import numpy as np
from mlx.utils import tree_map

from mlx_vlm import load
from mlx_vlm.models.switch_layers import QuantizedSwitchLinear

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
NATIVE = os.path.join(ROOT, "upstream", "Inkling-2-Mini")
PROJECTIONS = ("gate_proj", "up_proj", "down_proj")


# ----------------------------------------------------------------------------- model access
def load_native(fp32=False, path=NATIVE):
    model, processor = load(path)
    if fp32:
        to_fp32(model)
    mx.eval(model.parameters())
    return model, processor


def to_fp32(model):
    model.update(tree_map(
        lambda p: p.astype(mx.float32) if isinstance(p, mx.array) and p.dtype == mx.bfloat16 else p,
        model.parameters()))


def layers(model):
    return model.language_model.model.layers


def switch(model, layer):
    return layers(model)[layer].mlp.switch_mlp


def dense_expert(proj):
    """Exact float32 weights (experts, out, in) of a native or affine expert projection."""
    mode = proj.mode
    if mode == "nvfp4":
        w = mx.dequantize(proj.weight, proj.scales, None, group_size=16, bits=4, mode="nvfp4", dtype=mx.float32)
        scale = proj.global_scale.astype(mx.float32).reshape(-1, 1, 1)
        return w * scale if proj._scale_name == "weight_scale_2" else w / scale
    if mode == "mxfp8":
        return mx.dequantize(proj.weight, proj.scales, None, group_size=32, bits=8, mode="mxfp8", dtype=mx.float32)
    return mx.dequantize(proj.weight, proj.scales, proj.get("biases"), group_size=proj.group_size, bits=proj.bits,
                         mode=mode, dtype=mx.float32)


# ----------------------------------------------------------------------------- affine packing
def pack_codes(codes, bits):
    """Pack integer codes (..., n) into MLX's uint32 layout: a little-endian bit stream along the last axis."""
    n = codes.shape[-1]
    per = math.lcm(bits, 32) // bits          # codes per repeating unit
    words = math.lcm(bits, 32) // 32          # uint32 words per unit
    assert n % per == 0, (n, per)
    c = codes.astype(mx.uint32).reshape(*codes.shape[:-1], n // per, per)
    out = [None] * words
    for j in range(per):
        off = bits * j
        k, sh = off // 32, off % 32
        v = c[..., j]
        lo = mx.left_shift(v, mx.array(sh, mx.uint32)) if sh else v
        out[k] = lo if out[k] is None else mx.bitwise_or(out[k], lo)
        if sh + bits > 32:
            hi = mx.right_shift(v, mx.array(32 - sh, mx.uint32))
            out[k + 1] = hi if out[k + 1] is None else mx.bitwise_or(out[k + 1], hi)
    return mx.stack(out, axis=-1).reshape(*codes.shape[:-1], n // per * words)


def affine_switch(packed, scales, biases, bits, group_size):
    q = QuantizedSwitchLinear.__new__(QuantizedSwitchLinear)
    nn.Module.__init__(q)
    q.weight, q.scales, q.biases = packed, scales, biases
    q.group_size, q.bits, q.mode = group_size, bits, "affine"
    q.freeze()
    return q


def rtn_affine(w, bits, group_size, scale_dtype=mx.float16):
    """MLX's own round-to-nearest affine quantizer, with scales stored in `scale_dtype`."""
    wq, s, b = mx.quantize(w, group_size=group_size, bits=bits, mode="affine")
    return wq, s.astype(scale_dtype), b.astype(scale_dtype)


# ----------------------------------------------------------------------------- data
def render_ids(tok, rec):
    """Token ids of a recorded conversation (scenario record from gen_chats.py)."""
    import sys
    sys.path.insert(0, os.path.join(ROOT, "scripts"))
    from scenarios import TOOLS
    tools = [TOOLS[t] for t in rec.get("tools", [])] or None
    out = tok.apply_chat_template(rec["messages"], tools=tools, reasoning_effort=rec.get("effort"), tokenize=True)
    return list(out["input_ids"]) if hasattr(out, "keys") else list(out)


def load_sequences(path, split=None, max_len=1024):
    """Token sequences from a prepared JSON file: [{"name", "split", "kind", "ids"}]."""
    seqs = json.load(open(path))
    out = []
    for s in seqs:
        if split and s["split"] != split:
            continue
        ids = s["ids"]
        for start in range(0, len(ids), max_len):
            chunk = ids[start:start + max_len]
            if len(chunk) >= 16:
                out.append(dict(name=f"{s['name']}@{start}", kind=s["kind"], ids=chunk))
    return out


def logits_for(model, ids):
    out = model(mx.array([ids])).logits[0].astype(mx.float32)
    mx.eval(out)
    return out


def log_softmax(x):
    return x - mx.logsumexp(x, axis=-1, keepdims=True)


def compare(teacher_logits, student_logits, ids):
    """Per-sequence statistics; teacher and student are (T, V) float32 logits."""
    lt, ls = log_softmax(teacher_logits), log_softmax(student_logits)
    kl = (mx.exp(lt) * (lt - ls)).sum(-1)
    nxt = mx.array(ids[1:])
    nll_t = -mx.take_along_axis(lt[:-1], nxt[:, None], axis=-1)[:, 0]
    nll_s = -mx.take_along_axis(ls[:-1], nxt[:, None], axis=-1)[:, 0]
    agree = (mx.argmax(lt, -1) == mx.argmax(ls, -1)).astype(mx.float32)
    vals = [kl.sum(), kl.max(), agree.sum(), nll_t.sum(), nll_s.sum()]
    mx.eval(vals)
    return dict(n=len(ids), kl_sum=vals[0].item(), kl_max=vals[1].item(), agree=vals[2].item(),
                nll_t=vals[3].item(), nll_s=vals[4].item())


def summarize(rows):
    n = sum(r["n"] for r in rows)
    m = sum(r["n"] - 1 for r in rows)
    return dict(tokens=n, kl=sum(r["kl_sum"] for r in rows) / n, kl_max=max(r["kl_max"] for r in rows),
                top1=sum(r["agree"] for r in rows) / n, nll_teacher=sum(r["nll_t"] for r in rows) / m,
                nll_student=sum(r["nll_s"] for r in rows) / m)