Image-Text-to-Text
MLX
Safetensors
English
inkling2_loop
mlx-vlm
inkling2
Mixture of Experts
quantized
gptq
apple-silicon
tool-calling
agents
conversational
8-bit precision
Instructions to use VisualInference/Inkling-2-Mini-MLX-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use VisualInference/Inkling-2-Mini-MLX-4bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("VisualInference/Inkling-2-Mini-MLX-4bit") config = load_config("VisualInference/Inkling-2-Mini-MLX-4bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use VisualInference/Inkling-2-Mini-MLX-4bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-4bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "VisualInference/Inkling-2-Mini-MLX-4bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use VisualInference/Inkling-2-Mini-MLX-4bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-4bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default VisualInference/Inkling-2-Mini-MLX-4bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use VisualInference/Inkling-2-Mini-MLX-4bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-4bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "VisualInference/Inkling-2-Mini-MLX-4bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
File size: 8,961 Bytes
d19188c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 | """Error-compensated (GPTQ-style) affine requantization of Inkling 2 Mini experts, batched over experts.
For one projection W (experts, out, in) with per-expert input second moments H (experts, in, in), columns are
quantized left to right; after each column the remaining columns absorb the rounding error through the inverse
Hessian, so the layer output error on calibration inputs is minimised rather than the weight error. Scale and
bias of each 64-column group are chosen (with a small clipping search) from the already-updated weights and are
rounded to float16 before the codes are computed, so the stored parameters are exactly the ones optimised for.
Usage: gptq.py <hess_dir> <out_dir> --layers 0,1 --specs 3:64,2:64 [--damp 0.01] [--rtn]
"""
import os
import sys
import time
import mlx.core as mx
import numpy as np
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import qlab # noqa: E402
SHRINKS = (1.0, 0.9, 0.8, 0.7, 0.6)
def prepare_u(H, counts, damp):
"""Upper Cholesky factor of the inverse damped Hessian, (experts, in, in) float32, computed on CPU."""
e, d, _ = H.shape
diag = mx.diagonal(H, axis1=1, axis2=2)
mean = diag.mean(axis=1)
n = mx.array(np.maximum(counts, 1).astype(np.float32))
# Shrink towards the identity more strongly when an expert saw few samples (covariance is then rank-poor).
lam = mx.clip(0.5 * d / n, damp, 1.0) * mx.where(mean > 0, mean, 1.0)
never = mx.array((counts == 0).astype(np.float32))
Hd = H * (1 - never)[:, None, None] + mx.eye(d)[None] * (lam + never)[:, None, None]
Hd = Hd / mx.diagonal(Hd, axis1=1, axis2=2).mean(axis=1)[:, None, None]
mx.eval(Hd)
out = []
for i in range(0, e, 16):
inv = mx.linalg.inv(Hd[i:i + 16], stream=mx.cpu)
inv = (inv + inv.swapaxes(-1, -2)) * 0.5
u = mx.linalg.cholesky(inv, upper=True, stream=mx.cpu)
mx.eval(u)
out.append(u)
U = mx.concatenate(out, axis=0)
mx.eval(U)
return U, Hd
def _params(Wb, wt, levels, dtype):
"""Pick scale/bias per (expert, row) for one group by a clipping search. wt: (experts, g) column weights."""
lo, hi = Wb.min(axis=-1), Wb.max(axis=-1)
mid, half = (hi + lo) * 0.5, (hi - lo) * 0.5
best_err = best_s = best_b = None
for p in SHRINKS:
s = mx.maximum(2 * half * p / levels, 1e-8).astype(dtype).astype(mx.float32)
b = (mid - half * p).astype(dtype).astype(mx.float32)
q = mx.clip(mx.round((Wb - b[..., None]) / s[..., None]), 0, levels)
err = (mx.square(q * s[..., None] + b[..., None] - Wb) * wt[:, None, :]).sum(axis=-1)
if best_err is None:
best_err, best_s, best_b = err, s, b
else:
take = err < best_err
best_err = mx.where(take, err, best_err)
best_s = mx.where(take, s, best_s)
best_b = mx.where(take, b, best_b)
return best_s, best_b
def gptq_affine(W, U, bits, group, scale_dtype=mx.float16, compensate=True):
"""Returns (codes uint8 (E,out,in), scales (E,out,in/g), biases) for W (E,out,in) given U from prepare_u."""
e, out_dim, in_dim = W.shape
levels = float(2 ** bits - 1)
codes, scales, biases = [], [], []
W = W.astype(mx.float32)
tri = mx.triu(mx.ones((group, group)), k=1)
for i1 in range(0, in_dim, group):
i2 = i1 + group
Ub = U[:, i1:i2, i1:i2] # (E, g, g)
dg = mx.diagonal(Ub, axis1=1, axis2=2) # (E, g)
Wb = W[:, :, i1:i2]
s, b = _params(Wb, 1.0 / mx.square(dg), levels, scale_dtype)
Uoff = Ub * tri[None] # strictly upper part
cols, errs = [], []
cur = Wb
for j in range(group):
w = cur[:, :, j]
q = mx.clip(mx.round((w - b) / s), 0, levels)
cols.append(q)
if compensate:
err = (w - (q * s + b)) / dg[:, j][:, None]
errs.append(err)
cur = cur - err[:, :, None] * Uoff[:, j][:, None, :]
codes.append(mx.stack(cols, axis=-1).astype(mx.uint8))
scales.append(s.astype(scale_dtype)); biases.append(b.astype(scale_dtype))
if compensate and i2 < in_dim:
E = mx.stack(errs, axis=-1) # (E, out, g)
tail = W[:, :, i2:] - E @ U[:, i1:i2, i2:]
W = mx.concatenate([W[:, :, :i2], tail], axis=-1)
mx.eval(codes[-1], scales[-1], biases[-1], W)
return mx.concatenate(codes, axis=-1), mx.stack(scales, axis=-1), mx.stack(biases, axis=-1)
def dequant(codes, scales, biases, group):
e, o, n = codes.shape
c = codes.astype(mx.float32).reshape(e, o, n // group, group)
return (c * scales.astype(mx.float32)[..., None] + biases.astype(mx.float32)[..., None]).reshape(e, o, n)
def hessian_error(W, What, H):
"""sum_e tr(dW H dW^T) / sum_e tr(W H W^T): relative output error energy on the calibration inputs."""
num = den = 0.0
for i in range(0, W.shape[0], 16):
d = (What[i:i + 16] - W[i:i + 16]).astype(mx.float32)
w = W[i:i + 16].astype(mx.float32)
h = H[i:i + 16]
a = ((d @ h) * d).sum(); c = ((w @ h) * w).sum()
mx.eval(a, c)
num += a.item(); den += c.item()
return num / max(den, 1e-30)
def quantize_layer(model, layer, hess, specs, damp=0.01, compensate=True, log=print):
"""Quantize one layer's three projections for every (bits, group) spec. Returns {spec: {name: array}}, stats."""
sw = qlab.switch(model, layer)
counts = np.array(hess["n"])
gate, up, down = (qlab.dense_expert(getattr(sw, p)) for p in qlab.PROJECTIONS)
W13 = mx.concatenate([gate, up], axis=1)
mx.eval(W13, down)
half = gate.shape[1]
results, stats = {spec: {} for spec in specs}, {}
for name, W, H in (("w13", W13, hess["H13"]), ("w2", down, hess["H2"])):
t0 = time.time()
U, Hd = prepare_u(H, counts, damp)
t_lin = time.time() - t0
for spec in specs:
bits, group = spec
t1 = time.time()
codes, s, b = gptq_affine(W, U, bits, group, compensate=compensate)
err = hessian_error(W, dequant(codes, s, b, group), Hd)
wq, s0, b0 = qlab.rtn_affine(W, bits, group)
rtn = hessian_error(W, mx.dequantize(wq, s0.astype(mx.float32), b0.astype(mx.float32), group_size=group,
bits=bits, mode="affine", dtype=mx.float32), Hd)
stats[f"{name}:{bits}:{group}"] = dict(gptq=err, rtn=rtn)
log(f" L{layer:02d} {name} {bits}-bit g{group}: output error energy {err:.3e} (round-to-nearest {rtn:.3e}, "
f"ratio {rtn / max(err, 1e-30):.1f}x) linalg {t_lin:.0f}s quant {time.time() - t1:.0f}s")
packed = qlab.pack_codes(codes, bits)
mx.eval(packed)
if name == "w13":
results[spec].update({
"gate_proj.weight": packed[:, :half], "gate_proj.scales": s[:, :half], "gate_proj.biases": b[:, :half],
"up_proj.weight": packed[:, half:], "up_proj.scales": s[:, half:], "up_proj.biases": b[:, half:]})
else:
results[spec].update({"down_proj.weight": packed, "down_proj.scales": s, "down_proj.biases": b})
mx.eval(list(results[spec].values()))
del U, Hd
mx.clear_cache()
return results, stats
def main():
args = sys.argv[1:]
hess_dir, out_dir = args[:2]
layers, specs, damp, compensate = [], [], 0.01, True
rest = args[2:]
while rest:
if rest[0] == "--layers":
layers = [int(x) for x in rest[1].split(",")]; rest = rest[2:]
elif rest[0] == "--specs":
specs = [tuple(int(v) for v in s.split(":")) for s in rest[1].split(",")]; rest = rest[2:]
elif rest[0] == "--damp":
damp = float(rest[1]); rest = rest[2:]
elif rest[0] == "--rtn":
compensate = False; rest = rest[1:]
else:
raise SystemExit("unknown flag " + rest[0])
os.makedirs(out_dir, exist_ok=True)
from mlx_vlm import load
model, _ = load(qlab.NATIVE, lazy=True)
import json
all_stats = {}
for li in layers:
t0 = time.time()
hess = mx.load(os.path.join(hess_dir, f"hess_L{li:02d}.safetensors"))
results, stats = quantize_layer(model, li, hess, specs, damp, compensate)
for (bits, group), tensors in results.items():
mx.save_safetensors(os.path.join(out_dir, f"q_L{li:02d}_b{bits}g{group}.safetensors"), tensors)
all_stats[li] = stats
json.dump(all_stats, open(os.path.join(out_dir, f"stats_{layers[0]:02d}-{layers[-1]:02d}.json"), "w"), indent=1)
print(f"layer {li} done in {time.time() - t0:.0f}s; peak {mx.get_peak_memory() / 2**30:.1f} GB", flush=True)
del hess, results
mx.clear_cache()
if __name__ == "__main__":
main()
|