Image-Text-to-Text
MLX
Safetensors
English
inkling2_loop
mlx-vlm
inkling2
Mixture of Experts
quantized
gptq
apple-silicon
tool-calling
agents
conversational
8-bit precision
Instructions to use VisualInference/Inkling-2-Mini-MLX-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use VisualInference/Inkling-2-Mini-MLX-4bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("VisualInference/Inkling-2-Mini-MLX-4bit") config = load_config("VisualInference/Inkling-2-Mini-MLX-4bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use VisualInference/Inkling-2-Mini-MLX-4bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-4bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "VisualInference/Inkling-2-Mini-MLX-4bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use VisualInference/Inkling-2-Mini-MLX-4bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-4bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default VisualInference/Inkling-2-Mini-MLX-4bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use VisualInference/Inkling-2-Mini-MLX-4bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-4bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "VisualInference/Inkling-2-Mini-MLX-4bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download extras/scripts/gptq.py from VisualInference/Inkling-2-Mini-MLX-4bit: direct link, hf CLI and curl.
- Browser
- Download file 8.96 kB
-
https://huggingface.co/VisualInference/Inkling-2-Mini-MLX-4bit/resolve/main/extras/scripts/gptq.py
- Command line
-
hf download hf://VisualInference/Inkling-2-Mini-MLX-4bit/extras/scripts/gptq.py
-
curl -L -o gptq.py https://huggingface.co/VisualInference/Inkling-2-Mini-MLX-4bit/resolve/main/extras/scripts/gptq.py
8.96 kB
| """Error-compensated (GPTQ-style) affine requantization of Inkling 2 Mini experts, batched over experts. | |
| For one projection W (experts, out, in) with per-expert input second moments H (experts, in, in), columns are | |
| quantized left to right; after each column the remaining columns absorb the rounding error through the inverse | |
| Hessian, so the layer output error on calibration inputs is minimised rather than the weight error. Scale and | |
| bias of each 64-column group are chosen (with a small clipping search) from the already-updated weights and are | |
| rounded to float16 before the codes are computed, so the stored parameters are exactly the ones optimised for. | |
| Usage: gptq.py <hess_dir> <out_dir> --layers 0,1 --specs 3:64,2:64 [--damp 0.01] [--rtn] | |
| """ | |
| import os | |
| import sys | |
| import time | |
| import mlx.core as mx | |
| import numpy as np | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| import qlab # noqa: E402 | |
| SHRINKS = (1.0, 0.9, 0.8, 0.7, 0.6) | |
| def prepare_u(H, counts, damp): | |
| """Upper Cholesky factor of the inverse damped Hessian, (experts, in, in) float32, computed on CPU.""" | |
| e, d, _ = H.shape | |
| diag = mx.diagonal(H, axis1=1, axis2=2) | |
| mean = diag.mean(axis=1) | |
| n = mx.array(np.maximum(counts, 1).astype(np.float32)) | |
| # Shrink towards the identity more strongly when an expert saw few samples (covariance is then rank-poor). | |
| lam = mx.clip(0.5 * d / n, damp, 1.0) * mx.where(mean > 0, mean, 1.0) | |
| never = mx.array((counts == 0).astype(np.float32)) | |
| Hd = H * (1 - never)[:, None, None] + mx.eye(d)[None] * (lam + never)[:, None, None] | |
| Hd = Hd / mx.diagonal(Hd, axis1=1, axis2=2).mean(axis=1)[:, None, None] | |
| mx.eval(Hd) | |
| out = [] | |
| for i in range(0, e, 16): | |
| inv = mx.linalg.inv(Hd[i:i + 16], stream=mx.cpu) | |
| inv = (inv + inv.swapaxes(-1, -2)) * 0.5 | |
| u = mx.linalg.cholesky(inv, upper=True, stream=mx.cpu) | |
| mx.eval(u) | |
| out.append(u) | |
| U = mx.concatenate(out, axis=0) | |
| mx.eval(U) | |
| return U, Hd | |
| def _params(Wb, wt, levels, dtype): | |
| """Pick scale/bias per (expert, row) for one group by a clipping search. wt: (experts, g) column weights.""" | |
| lo, hi = Wb.min(axis=-1), Wb.max(axis=-1) | |
| mid, half = (hi + lo) * 0.5, (hi - lo) * 0.5 | |
| best_err = best_s = best_b = None | |
| for p in SHRINKS: | |
| s = mx.maximum(2 * half * p / levels, 1e-8).astype(dtype).astype(mx.float32) | |
| b = (mid - half * p).astype(dtype).astype(mx.float32) | |
| q = mx.clip(mx.round((Wb - b[..., None]) / s[..., None]), 0, levels) | |
| err = (mx.square(q * s[..., None] + b[..., None] - Wb) * wt[:, None, :]).sum(axis=-1) | |
| if best_err is None: | |
| best_err, best_s, best_b = err, s, b | |
| else: | |
| take = err < best_err | |
| best_err = mx.where(take, err, best_err) | |
| best_s = mx.where(take, s, best_s) | |
| best_b = mx.where(take, b, best_b) | |
| return best_s, best_b | |
| def gptq_affine(W, U, bits, group, scale_dtype=mx.float16, compensate=True): | |
| """Returns (codes uint8 (E,out,in), scales (E,out,in/g), biases) for W (E,out,in) given U from prepare_u.""" | |
| e, out_dim, in_dim = W.shape | |
| levels = float(2 ** bits - 1) | |
| codes, scales, biases = [], [], [] | |
| W = W.astype(mx.float32) | |
| tri = mx.triu(mx.ones((group, group)), k=1) | |
| for i1 in range(0, in_dim, group): | |
| i2 = i1 + group | |
| Ub = U[:, i1:i2, i1:i2] # (E, g, g) | |
| dg = mx.diagonal(Ub, axis1=1, axis2=2) # (E, g) | |
| Wb = W[:, :, i1:i2] | |
| s, b = _params(Wb, 1.0 / mx.square(dg), levels, scale_dtype) | |
| Uoff = Ub * tri[None] # strictly upper part | |
| cols, errs = [], [] | |
| cur = Wb | |
| for j in range(group): | |
| w = cur[:, :, j] | |
| q = mx.clip(mx.round((w - b) / s), 0, levels) | |
| cols.append(q) | |
| if compensate: | |
| err = (w - (q * s + b)) / dg[:, j][:, None] | |
| errs.append(err) | |
| cur = cur - err[:, :, None] * Uoff[:, j][:, None, :] | |
| codes.append(mx.stack(cols, axis=-1).astype(mx.uint8)) | |
| scales.append(s.astype(scale_dtype)); biases.append(b.astype(scale_dtype)) | |
| if compensate and i2 < in_dim: | |
| E = mx.stack(errs, axis=-1) # (E, out, g) | |
| tail = W[:, :, i2:] - E @ U[:, i1:i2, i2:] | |
| W = mx.concatenate([W[:, :, :i2], tail], axis=-1) | |
| mx.eval(codes[-1], scales[-1], biases[-1], W) | |
| return mx.concatenate(codes, axis=-1), mx.stack(scales, axis=-1), mx.stack(biases, axis=-1) | |
| def dequant(codes, scales, biases, group): | |
| e, o, n = codes.shape | |
| c = codes.astype(mx.float32).reshape(e, o, n // group, group) | |
| return (c * scales.astype(mx.float32)[..., None] + biases.astype(mx.float32)[..., None]).reshape(e, o, n) | |
| def hessian_error(W, What, H): | |
| """sum_e tr(dW H dW^T) / sum_e tr(W H W^T): relative output error energy on the calibration inputs.""" | |
| num = den = 0.0 | |
| for i in range(0, W.shape[0], 16): | |
| d = (What[i:i + 16] - W[i:i + 16]).astype(mx.float32) | |
| w = W[i:i + 16].astype(mx.float32) | |
| h = H[i:i + 16] | |
| a = ((d @ h) * d).sum(); c = ((w @ h) * w).sum() | |
| mx.eval(a, c) | |
| num += a.item(); den += c.item() | |
| return num / max(den, 1e-30) | |
| def quantize_layer(model, layer, hess, specs, damp=0.01, compensate=True, log=print): | |
| """Quantize one layer's three projections for every (bits, group) spec. Returns {spec: {name: array}}, stats.""" | |
| sw = qlab.switch(model, layer) | |
| counts = np.array(hess["n"]) | |
| gate, up, down = (qlab.dense_expert(getattr(sw, p)) for p in qlab.PROJECTIONS) | |
| W13 = mx.concatenate([gate, up], axis=1) | |
| mx.eval(W13, down) | |
| half = gate.shape[1] | |
| results, stats = {spec: {} for spec in specs}, {} | |
| for name, W, H in (("w13", W13, hess["H13"]), ("w2", down, hess["H2"])): | |
| t0 = time.time() | |
| U, Hd = prepare_u(H, counts, damp) | |
| t_lin = time.time() - t0 | |
| for spec in specs: | |
| bits, group = spec | |
| t1 = time.time() | |
| codes, s, b = gptq_affine(W, U, bits, group, compensate=compensate) | |
| err = hessian_error(W, dequant(codes, s, b, group), Hd) | |
| wq, s0, b0 = qlab.rtn_affine(W, bits, group) | |
| rtn = hessian_error(W, mx.dequantize(wq, s0.astype(mx.float32), b0.astype(mx.float32), group_size=group, | |
| bits=bits, mode="affine", dtype=mx.float32), Hd) | |
| stats[f"{name}:{bits}:{group}"] = dict(gptq=err, rtn=rtn) | |
| log(f" L{layer:02d} {name} {bits}-bit g{group}: output error energy {err:.3e} (round-to-nearest {rtn:.3e}, " | |
| f"ratio {rtn / max(err, 1e-30):.1f}x) linalg {t_lin:.0f}s quant {time.time() - t1:.0f}s") | |
| packed = qlab.pack_codes(codes, bits) | |
| mx.eval(packed) | |
| if name == "w13": | |
| results[spec].update({ | |
| "gate_proj.weight": packed[:, :half], "gate_proj.scales": s[:, :half], "gate_proj.biases": b[:, :half], | |
| "up_proj.weight": packed[:, half:], "up_proj.scales": s[:, half:], "up_proj.biases": b[:, half:]}) | |
| else: | |
| results[spec].update({"down_proj.weight": packed, "down_proj.scales": s, "down_proj.biases": b}) | |
| mx.eval(list(results[spec].values())) | |
| del U, Hd | |
| mx.clear_cache() | |
| return results, stats | |
| def main(): | |
| args = sys.argv[1:] | |
| hess_dir, out_dir = args[:2] | |
| layers, specs, damp, compensate = [], [], 0.01, True | |
| rest = args[2:] | |
| while rest: | |
| if rest[0] == "--layers": | |
| layers = [int(x) for x in rest[1].split(",")]; rest = rest[2:] | |
| elif rest[0] == "--specs": | |
| specs = [tuple(int(v) for v in s.split(":")) for s in rest[1].split(",")]; rest = rest[2:] | |
| elif rest[0] == "--damp": | |
| damp = float(rest[1]); rest = rest[2:] | |
| elif rest[0] == "--rtn": | |
| compensate = False; rest = rest[1:] | |
| else: | |
| raise SystemExit("unknown flag " + rest[0]) | |
| os.makedirs(out_dir, exist_ok=True) | |
| from mlx_vlm import load | |
| model, _ = load(qlab.NATIVE, lazy=True) | |
| import json | |
| all_stats = {} | |
| for li in layers: | |
| t0 = time.time() | |
| hess = mx.load(os.path.join(hess_dir, f"hess_L{li:02d}.safetensors")) | |
| results, stats = quantize_layer(model, li, hess, specs, damp, compensate) | |
| for (bits, group), tensors in results.items(): | |
| mx.save_safetensors(os.path.join(out_dir, f"q_L{li:02d}_b{bits}g{group}.safetensors"), tensors) | |
| all_stats[li] = stats | |
| json.dump(all_stats, open(os.path.join(out_dir, f"stats_{layers[0]:02d}-{layers[-1]:02d}.json"), "w"), indent=1) | |
| print(f"layer {li} done in {time.time() - t0:.0f}s; peak {mx.get_peak_memory() / 2**30:.1f} GB", flush=True) | |
| del hess, results | |
| mx.clear_cache() | |
| if __name__ == "__main__": | |
| main() | |