Image-Text-to-Text
MLX
Safetensors
English
inkling2_loop
mlx-vlm
inkling2
Mixture of Experts
quantized
gptq
apple-silicon
tool-calling
agents
conversational
8-bit precision
Instructions to use VisualInference/Inkling-2-Mini-MLX-3bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use VisualInference/Inkling-2-Mini-MLX-3bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("VisualInference/Inkling-2-Mini-MLX-3bit") config = load_config("VisualInference/Inkling-2-Mini-MLX-3bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use VisualInference/Inkling-2-Mini-MLX-3bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-3bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "VisualInference/Inkling-2-Mini-MLX-3bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use VisualInference/Inkling-2-Mini-MLX-3bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-3bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default VisualInference/Inkling-2-Mini-MLX-3bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use VisualInference/Inkling-2-Mini-MLX-3bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-3bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "VisualInference/Inkling-2-Mini-MLX-3bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download extras/scripts/calib_capture.py from VisualInference/Inkling-2-Mini-MLX-3bit: direct link, hf CLI and curl.
- Browser
- Download file 3.4 kB
-
https://huggingface.co/VisualInference/Inkling-2-Mini-MLX-3bit/resolve/main/extras/scripts/calib_capture.py
- Command line
-
hf download hf://VisualInference/Inkling-2-Mini-MLX-3bit/extras/scripts/calib_capture.py
-
curl -L -o calib_capture.py https://huggingface.co/VisualInference/Inkling-2-Mini-MLX-3bit/resolve/main/extras/scripts/calib_capture.py
3.4 kB
| """Accumulate per-expert input second moments (GPTQ Hessians) for a set of layers. | |
| For every routed (token, expert) pair of a layer visit: | |
| H13[e] += x x^T (input of gate_proj / up_proj, dim 2304) | |
| H2[e] += a a^T (input of down_proj, a = softcapped_swiglu(up_e x, gate_e x), dim 1152) | |
| Looped layers accumulate over all three visits. Usage: calib_capture.py <sequences.json> <out_dir> <layer,layer,...> | |
| """ | |
| import os, sys, time | |
| import numpy as np | |
| import mlx.core as mx | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| import qlab # noqa: E402 | |
| from mlx_vlm.models.inkling2 import language as L # noqa: E402 | |
| seq_path, out_dir, layer_arg = sys.argv[1:4] | |
| want = [int(x) for x in layer_arg.split(",")] | |
| os.makedirs(out_dir, exist_ok=True) | |
| model, _ = qlab.load_native() | |
| seqs = qlab.load_sequences(seq_path, split="calib", max_len=1024) | |
| state = {} | |
| for li in want: | |
| moe = qlab.layers(model)[li].mlp | |
| e, d, h = moe.gate_weight.shape[0], moe.gate_weight.shape[1], moe.switch_mlp.down_proj.input_dims | |
| state[id(moe)] = dict(layer=li, H13=[mx.zeros((d, d)) for _ in range(e)], H2=[mx.zeros((h, h)) for _ in range(e)], | |
| n=np.zeros(e, np.int64)) | |
| orig = L.SparseMoE.__call__ | |
| def patched(self, x): | |
| st = state.get(id(self)) | |
| if st is not None: | |
| indices, _ = L._route(x, self.gate_weight, self.e_score_correction_bias, self.top_k, self.route_scale) | |
| X = x.reshape(-1, x.shape[-1]) | |
| idx = np.array(indices.reshape(-1, indices.shape[-1])) | |
| flat = idx.reshape(-1) | |
| order = np.argsort(flat, kind="stable") | |
| tok_of = order // idx.shape[1] | |
| bounds = np.searchsorted(flat[order], np.arange(len(st["H13"]) + 1)) | |
| sw = self.switch_mlp | |
| H13, H2 = st["H13"], st["H2"] | |
| touched = [] | |
| for e in range(len(bounds) - 1): | |
| lo, hi = int(bounds[e]), int(bounds[e + 1]) | |
| if hi == lo: | |
| continue | |
| rows = mx.array(tok_of[lo:hi]) | |
| xe = X[rows] # (n, D), model dtype | |
| ei = mx.full((hi - lo, 1), e, dtype=mx.uint32) | |
| xin = xe[:, None, None, :] | |
| up = sw.up_proj(xin, ei) | |
| gate = sw.gate_proj(xin, ei) | |
| a = sw.activation(up, gate).reshape(hi - lo, -1).astype(mx.float32) | |
| xf = xe.astype(mx.float32) | |
| H13[e] = H13[e] + xf.T @ xf | |
| H2[e] = H2[e] + a.T @ a | |
| touched += [H13[e], H2[e]] | |
| st["n"][e] += hi - lo | |
| mx.eval(touched) | |
| return orig(self, x) | |
| L.SparseMoE.__call__ = patched | |
| t0 = time.time(); ntok = 0 | |
| for i, s in enumerate(seqs): | |
| out = model(mx.array([s["ids"]])) | |
| mx.eval(out.logits[0, -1]) | |
| ntok += len(s["ids"]) | |
| mx.clear_cache() | |
| if i % 10 == 0: | |
| print(f"seq {i}/{len(seqs)} tokens {ntok} elapsed {time.time() - t0:.0f}s peak {mx.get_peak_memory() / 2**30:.1f} GB", flush=True) | |
| for st in state.values(): | |
| li = st["layer"] | |
| mx.save_safetensors(os.path.join(out_dir, f"hess_L{li:02d}.safetensors"), | |
| {"H13": mx.stack(st["H13"]), "H2": mx.stack(st["H2"]), "n": mx.array(st["n"])}) | |
| print(f"layer {li}: routed pairs {int(st['n'].sum())} min/median/max per expert " | |
| f"{int(st['n'].min())}/{int(np.median(st['n']))}/{int(st['n'].max())}", flush=True) | |
| print("done", round(time.time() - t0), "s tokens", ntok) | |