Image-Text-to-Text
MLX
Safetensors
English
inkling2_loop
mlx-vlm
inkling2
Mixture of Experts
quantized
gptq
apple-silicon
tool-calling
agents
conversational
8-bit precision
Instructions to use VisualInference/Inkling-2-Mini-MLX-2bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use VisualInference/Inkling-2-Mini-MLX-2bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("VisualInference/Inkling-2-Mini-MLX-2bit") config = load_config("VisualInference/Inkling-2-Mini-MLX-2bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use VisualInference/Inkling-2-Mini-MLX-2bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-2bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "VisualInference/Inkling-2-Mini-MLX-2bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use VisualInference/Inkling-2-Mini-MLX-2bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-2bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default VisualInference/Inkling-2-Mini-MLX-2bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use VisualInference/Inkling-2-Mini-MLX-2bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-2bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "VisualInference/Inkling-2-Mini-MLX-2bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
File size: 4,349 Bytes
031489f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | """Apply a quantization recipe to the loaded native model (in memory), and describe it for config.json.
A recipe is a dict:
{"experts": {"default": SPEC, "layers": {"20": SPEC, ...}}, SPEC = "native" | "rtn:BITS:GROUP" | "gptq:BITS:GROUP"
"dense": {"attn": Q, "embed": Q, "head": Q, "vision": Q, "draft": Q}} Q = null (keep float) | [BITS, GROUP]
"gptq" shards are read from `qdir` (files q_LNN_bBgG.safetensors written by gptq.py).
"""
import os
import mlx.core as mx
import mlx.nn as nn
import qlab
ATTN_BIG = ("q_proj", "k_proj", "v_proj", "r_proj", "o_proj", "gate_proj")
def expert_spec(recipe, layer):
ex = recipe.get("experts", {})
return ex.get("layers", {}).get(str(layer), ex.get("default", "native"))
def dense_group(path, module):
"""Which recipe group a dense module belongs to, or None if it must stay in floating point."""
if ".mlp.switch_mlp." in path:
return None
if path == "language_model.model.embed_tokens":
return "embed"
if path == "language_model.lm_head":
return "head"
if path.startswith("language_model.model.layers.") and ".self_attn." in path:
leaf = path.rsplit(".", 1)[-1]
# sink_proj (24 rows) and the dynamic-convolution tap projections are tiny and stay bf16.
return "attn" if leaf in ATTN_BIG and ".k_sconv." not in path and ".v_sconv." not in path else None
if path.startswith("vision_tower."):
return "vision"
return None
def apply_experts(model, recipe, qdir=None, log=None):
"""Replace expert projections according to the recipe. Returns {module_path: quantization dict}."""
overrides = {}
act_dtype = model.language_model.model.embed_norm.weight.dtype
for li, layer in enumerate(qlab.layers(model)):
spec = expert_spec(recipe, li)
sw = layer.mlp.switch_mlp
base = f"language_model.model.layers.{li}.mlp.switch_mlp."
if spec == "native":
for p in qlab.PROJECTIONS:
proj = getattr(sw, p)
overrides[base + p] = {"group_size": proj.group_size, "bits": proj.bits, "mode": proj.mode}
continue
kind, bits, group = spec.split(":")
bits, group = int(bits), int(group)
if kind == "gptq":
shard = mx.load(os.path.join(qdir, f"q_L{li:02d}_b{bits}g{group}.safetensors"))
for p in qlab.PROJECTIONS:
if kind == "gptq":
packed, s, b = shard[p + ".weight"], shard[p + ".scales"], shard[p + ".biases"]
elif kind == "rtn":
packed, s, b = qlab.rtn_affine(qlab.dense_expert(getattr(sw, p)), bits, group)
else:
raise ValueError(spec)
# Scales and offsets are stored in the activation dtype. MLX's expert kernel (gather_qmm) promotes to
# float32 and runs about three times slower when their dtype differs from the input's.
s, b = s.astype(act_dtype), b.astype(act_dtype)
mx.eval(packed, s, b)
setattr(sw, p, qlab.affine_switch(packed, s, b, bits, group))
overrides[base + p] = {"group_size": group, "bits": bits, "mode": "affine"}
mx.clear_cache()
if log:
log(f"layer {li}: experts -> {spec}")
return overrides
def apply_dense(model, recipe):
"""Affine-quantize dense groups. Returns {module_path: quantization dict}."""
dense = recipe.get("dense", {}) or {}
overrides = {}
def predicate(path, module):
if not hasattr(module, "to_quantized") or not hasattr(module, "weight"):
return False
q = dense.get(dense_group(path, module))
if not q:
return False
bits, group = q
if module.weight.shape[-1] % group != 0:
return False
overrides[path] = {"group_size": group, "bits": bits, "mode": "affine"}
return {"group_size": group, "bits": bits, "mode": "affine"}
nn.quantize(model, class_predicate=predicate)
return overrides
def apply(model, recipe, qdir=None, log=None):
overrides = apply_experts(model, recipe, qdir, log)
overrides.update(apply_dense(model, recipe))
mx.eval(model.parameters())
return overrides
def nbytes(model):
from mlx.utils import tree_flatten
return sum(v.nbytes for _, v in tree_flatten(model.parameters()))
|