Image-Text-to-Text
MLX
Safetensors
English
inkling2_loop
mlx-vlm
inkling2
Mixture of Experts
quantized
gptq
apple-silicon
tool-calling
agents
conversational
8-bit precision
Instructions to use VisualInference/Inkling-2-Mini-MLX-2bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use VisualInference/Inkling-2-Mini-MLX-2bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("VisualInference/Inkling-2-Mini-MLX-2bit") config = load_config("VisualInference/Inkling-2-Mini-MLX-2bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use VisualInference/Inkling-2-Mini-MLX-2bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-2bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "VisualInference/Inkling-2-Mini-MLX-2bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use VisualInference/Inkling-2-Mini-MLX-2bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-2bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default VisualInference/Inkling-2-Mini-MLX-2bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use VisualInference/Inkling-2-Mini-MLX-2bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-2bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "VisualInference/Inkling-2-Mini-MLX-2bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
File size: 3,401 Bytes
031489f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 | """Accumulate per-expert input second moments (GPTQ Hessians) for a set of layers.
For every routed (token, expert) pair of a layer visit:
H13[e] += x x^T (input of gate_proj / up_proj, dim 2304)
H2[e] += a a^T (input of down_proj, a = softcapped_swiglu(up_e x, gate_e x), dim 1152)
Looped layers accumulate over all three visits. Usage: calib_capture.py <sequences.json> <out_dir> <layer,layer,...>
"""
import os, sys, time
import numpy as np
import mlx.core as mx
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import qlab # noqa: E402
from mlx_vlm.models.inkling2 import language as L # noqa: E402
seq_path, out_dir, layer_arg = sys.argv[1:4]
want = [int(x) for x in layer_arg.split(",")]
os.makedirs(out_dir, exist_ok=True)
model, _ = qlab.load_native()
seqs = qlab.load_sequences(seq_path, split="calib", max_len=1024)
state = {}
for li in want:
moe = qlab.layers(model)[li].mlp
e, d, h = moe.gate_weight.shape[0], moe.gate_weight.shape[1], moe.switch_mlp.down_proj.input_dims
state[id(moe)] = dict(layer=li, H13=[mx.zeros((d, d)) for _ in range(e)], H2=[mx.zeros((h, h)) for _ in range(e)],
n=np.zeros(e, np.int64))
orig = L.SparseMoE.__call__
def patched(self, x):
st = state.get(id(self))
if st is not None:
indices, _ = L._route(x, self.gate_weight, self.e_score_correction_bias, self.top_k, self.route_scale)
X = x.reshape(-1, x.shape[-1])
idx = np.array(indices.reshape(-1, indices.shape[-1]))
flat = idx.reshape(-1)
order = np.argsort(flat, kind="stable")
tok_of = order // idx.shape[1]
bounds = np.searchsorted(flat[order], np.arange(len(st["H13"]) + 1))
sw = self.switch_mlp
H13, H2 = st["H13"], st["H2"]
touched = []
for e in range(len(bounds) - 1):
lo, hi = int(bounds[e]), int(bounds[e + 1])
if hi == lo:
continue
rows = mx.array(tok_of[lo:hi])
xe = X[rows] # (n, D), model dtype
ei = mx.full((hi - lo, 1), e, dtype=mx.uint32)
xin = xe[:, None, None, :]
up = sw.up_proj(xin, ei)
gate = sw.gate_proj(xin, ei)
a = sw.activation(up, gate).reshape(hi - lo, -1).astype(mx.float32)
xf = xe.astype(mx.float32)
H13[e] = H13[e] + xf.T @ xf
H2[e] = H2[e] + a.T @ a
touched += [H13[e], H2[e]]
st["n"][e] += hi - lo
mx.eval(touched)
return orig(self, x)
L.SparseMoE.__call__ = patched
t0 = time.time(); ntok = 0
for i, s in enumerate(seqs):
out = model(mx.array([s["ids"]]))
mx.eval(out.logits[0, -1])
ntok += len(s["ids"])
mx.clear_cache()
if i % 10 == 0:
print(f"seq {i}/{len(seqs)} tokens {ntok} elapsed {time.time() - t0:.0f}s peak {mx.get_peak_memory() / 2**30:.1f} GB", flush=True)
for st in state.values():
li = st["layer"]
mx.save_safetensors(os.path.join(out_dir, f"hess_L{li:02d}.safetensors"),
{"H13": mx.stack(st["H13"]), "H2": mx.stack(st["H2"]), "n": mx.array(st["n"])})
print(f"layer {li}: routed pairs {int(st['n'].sum())} min/median/max per expert "
f"{int(st['n'].min())}/{int(np.median(st['n']))}/{int(st['n'].max())}", flush=True)
print("done", round(time.time() - t0), "s tokens", ntok)
|