Image-Text-to-Text
MLX
Safetensors
English
inkling2_loop
mlx-vlm
inkling2
Mixture of Experts
quantized
gptq
apple-silicon
tool-calling
agents
conversational
8-bit precision
Instructions to use VisualInference/Inkling-2-Mini-MLX-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use VisualInference/Inkling-2-Mini-MLX-4bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("VisualInference/Inkling-2-Mini-MLX-4bit") config = load_config("VisualInference/Inkling-2-Mini-MLX-4bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use VisualInference/Inkling-2-Mini-MLX-4bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-4bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "VisualInference/Inkling-2-Mini-MLX-4bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use VisualInference/Inkling-2-Mini-MLX-4bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-4bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default VisualInference/Inkling-2-Mini-MLX-4bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use VisualInference/Inkling-2-Mini-MLX-4bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-4bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "VisualInference/Inkling-2-Mini-MLX-4bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download extras/scripts/qlab.py from VisualInference/Inkling-2-Mini-MLX-4bit: direct link, hf CLI and curl.
- Browser
- Download file 5.8 kB
-
https://huggingface.co/VisualInference/Inkling-2-Mini-MLX-4bit/resolve/main/extras/scripts/qlab.py
- Command line
-
hf download hf://VisualInference/Inkling-2-Mini-MLX-4bit/extras/scripts/qlab.py
-
curl -L -o qlab.py https://huggingface.co/VisualInference/Inkling-2-Mini-MLX-4bit/resolve/main/extras/scripts/qlab.py
5.8 kB
| """Shared helpers for the Inkling 2 Mini quantization work (mlx-vlm layout).""" | |
| import json | |
| import math | |
| import os | |
| import mlx.core as mx | |
| import mlx.nn as nn | |
| import numpy as np | |
| from mlx.utils import tree_map | |
| from mlx_vlm import load | |
| from mlx_vlm.models.switch_layers import QuantizedSwitchLinear | |
| ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) | |
| NATIVE = os.path.join(ROOT, "upstream", "Inkling-2-Mini") | |
| PROJECTIONS = ("gate_proj", "up_proj", "down_proj") | |
| # ----------------------------------------------------------------------------- model access | |
| def load_native(fp32=False, path=NATIVE): | |
| model, processor = load(path) | |
| if fp32: | |
| to_fp32(model) | |
| mx.eval(model.parameters()) | |
| return model, processor | |
| def to_fp32(model): | |
| model.update(tree_map( | |
| lambda p: p.astype(mx.float32) if isinstance(p, mx.array) and p.dtype == mx.bfloat16 else p, | |
| model.parameters())) | |
| def layers(model): | |
| return model.language_model.model.layers | |
| def switch(model, layer): | |
| return layers(model)[layer].mlp.switch_mlp | |
| def dense_expert(proj): | |
| """Exact float32 weights (experts, out, in) of a native or affine expert projection.""" | |
| mode = proj.mode | |
| if mode == "nvfp4": | |
| w = mx.dequantize(proj.weight, proj.scales, None, group_size=16, bits=4, mode="nvfp4", dtype=mx.float32) | |
| scale = proj.global_scale.astype(mx.float32).reshape(-1, 1, 1) | |
| return w * scale if proj._scale_name == "weight_scale_2" else w / scale | |
| if mode == "mxfp8": | |
| return mx.dequantize(proj.weight, proj.scales, None, group_size=32, bits=8, mode="mxfp8", dtype=mx.float32) | |
| return mx.dequantize(proj.weight, proj.scales, proj.get("biases"), group_size=proj.group_size, bits=proj.bits, | |
| mode=mode, dtype=mx.float32) | |
| # ----------------------------------------------------------------------------- affine packing | |
| def pack_codes(codes, bits): | |
| """Pack integer codes (..., n) into MLX's uint32 layout: a little-endian bit stream along the last axis.""" | |
| n = codes.shape[-1] | |
| per = math.lcm(bits, 32) // bits # codes per repeating unit | |
| words = math.lcm(bits, 32) // 32 # uint32 words per unit | |
| assert n % per == 0, (n, per) | |
| c = codes.astype(mx.uint32).reshape(*codes.shape[:-1], n // per, per) | |
| out = [None] * words | |
| for j in range(per): | |
| off = bits * j | |
| k, sh = off // 32, off % 32 | |
| v = c[..., j] | |
| lo = mx.left_shift(v, mx.array(sh, mx.uint32)) if sh else v | |
| out[k] = lo if out[k] is None else mx.bitwise_or(out[k], lo) | |
| if sh + bits > 32: | |
| hi = mx.right_shift(v, mx.array(32 - sh, mx.uint32)) | |
| out[k + 1] = hi if out[k + 1] is None else mx.bitwise_or(out[k + 1], hi) | |
| return mx.stack(out, axis=-1).reshape(*codes.shape[:-1], n // per * words) | |
| def affine_switch(packed, scales, biases, bits, group_size): | |
| q = QuantizedSwitchLinear.__new__(QuantizedSwitchLinear) | |
| nn.Module.__init__(q) | |
| q.weight, q.scales, q.biases = packed, scales, biases | |
| q.group_size, q.bits, q.mode = group_size, bits, "affine" | |
| q.freeze() | |
| return q | |
| def rtn_affine(w, bits, group_size, scale_dtype=mx.float16): | |
| """MLX's own round-to-nearest affine quantizer, with scales stored in `scale_dtype`.""" | |
| wq, s, b = mx.quantize(w, group_size=group_size, bits=bits, mode="affine") | |
| return wq, s.astype(scale_dtype), b.astype(scale_dtype) | |
| # ----------------------------------------------------------------------------- data | |
| def render_ids(tok, rec): | |
| """Token ids of a recorded conversation (scenario record from gen_chats.py).""" | |
| import sys | |
| sys.path.insert(0, os.path.join(ROOT, "scripts")) | |
| from scenarios import TOOLS | |
| tools = [TOOLS[t] for t in rec.get("tools", [])] or None | |
| out = tok.apply_chat_template(rec["messages"], tools=tools, reasoning_effort=rec.get("effort"), tokenize=True) | |
| return list(out["input_ids"]) if hasattr(out, "keys") else list(out) | |
| def load_sequences(path, split=None, max_len=1024): | |
| """Token sequences from a prepared JSON file: [{"name", "split", "kind", "ids"}].""" | |
| seqs = json.load(open(path)) | |
| out = [] | |
| for s in seqs: | |
| if split and s["split"] != split: | |
| continue | |
| ids = s["ids"] | |
| for start in range(0, len(ids), max_len): | |
| chunk = ids[start:start + max_len] | |
| if len(chunk) >= 16: | |
| out.append(dict(name=f"{s['name']}@{start}", kind=s["kind"], ids=chunk)) | |
| return out | |
| def logits_for(model, ids): | |
| out = model(mx.array([ids])).logits[0].astype(mx.float32) | |
| mx.eval(out) | |
| return out | |
| def log_softmax(x): | |
| return x - mx.logsumexp(x, axis=-1, keepdims=True) | |
| def compare(teacher_logits, student_logits, ids): | |
| """Per-sequence statistics; teacher and student are (T, V) float32 logits.""" | |
| lt, ls = log_softmax(teacher_logits), log_softmax(student_logits) | |
| kl = (mx.exp(lt) * (lt - ls)).sum(-1) | |
| nxt = mx.array(ids[1:]) | |
| nll_t = -mx.take_along_axis(lt[:-1], nxt[:, None], axis=-1)[:, 0] | |
| nll_s = -mx.take_along_axis(ls[:-1], nxt[:, None], axis=-1)[:, 0] | |
| agree = (mx.argmax(lt, -1) == mx.argmax(ls, -1)).astype(mx.float32) | |
| vals = [kl.sum(), kl.max(), agree.sum(), nll_t.sum(), nll_s.sum()] | |
| mx.eval(vals) | |
| return dict(n=len(ids), kl_sum=vals[0].item(), kl_max=vals[1].item(), agree=vals[2].item(), | |
| nll_t=vals[3].item(), nll_s=vals[4].item()) | |
| def summarize(rows): | |
| n = sum(r["n"] for r in rows) | |
| m = sum(r["n"] - 1 for r in rows) | |
| return dict(tokens=n, kl=sum(r["kl_sum"] for r in rows) / n, kl_max=max(r["kl_max"] for r in rows), | |
| top1=sum(r["agree"] for r in rows) / n, nll_teacher=sum(r["nll_t"] for r in rows) / m, | |
| nll_student=sum(r["nll_s"] for r in rows) / m) | |