Image-Text-to-Text
MLX
Safetensors
English
inkling2_loop
mlx-vlm
inkling2
Mixture of Experts
quantized
gptq
apple-silicon
tool-calling
agents
conversational
8-bit precision
Instructions to use VisualInference/Inkling-2-Mini-MLX-3bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use VisualInference/Inkling-2-Mini-MLX-3bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("VisualInference/Inkling-2-Mini-MLX-3bit") config = load_config("VisualInference/Inkling-2-Mini-MLX-3bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use VisualInference/Inkling-2-Mini-MLX-3bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-3bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "VisualInference/Inkling-2-Mini-MLX-3bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use VisualInference/Inkling-2-Mini-MLX-3bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-3bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default VisualInference/Inkling-2-Mini-MLX-3bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use VisualInference/Inkling-2-Mini-MLX-3bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-3bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "VisualInference/Inkling-2-Mini-MLX-3bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
File size: 3,733 Bytes
e0b70af | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 | """Build a publishable mlx-vlm checkpoint directory from a recipe.
Usage: build_variant.py <recipes.json> <recipe_name> <out_dir> [--qdir q/shards]
Writes weights (mlx-vlm layout), config.json with explicit per-module quantization, tokenizer and processor files,
the bundled DSpark drafter (mtp.safetensors, unchanged) and a build manifest; then reloads the directory and checks
that its logits equal the in-memory model's.
"""
import hashlib
import json
import os
import shutil
import sys
import mlx.core as mx
import numpy as np
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
import qlab # noqa: E402
import recipes as R # noqa: E402
from mlx_vlm import load # noqa: E402
from mlx_vlm.utils import save_config, save_weights # noqa: E402
COPY = ("tokenizer.json", "tokenizer_config.json", "special_tokens_map.json", "chat_template.jinja",
"processor_config.json", "generation_config.json", "mtp.safetensors")
UPSTREAM_REVISION = "ed40e7da625f594fc9c4990f5f66be24873c22d7"
def sha256(path):
h = hashlib.sha256()
with open(path, "rb") as f:
while chunk := f.read(16 << 20):
h.update(chunk)
return h.hexdigest()
def main():
args = sys.argv[1:]
qdir = os.path.join(qlab.ROOT, "q", "shards")
if "--qdir" in args:
i = args.index("--qdir"); qdir = args[i + 1]; del args[i:i + 2]
recipe_path, name, out_dir = args[:3]
recipe = json.load(open(recipe_path))[name]
os.makedirs(out_dir, exist_ok=True)
model, processor = qlab.load_native()
probe = json.load(open(os.path.join(qlab.ROOT, "parity/ref_fp32/tool_roundtrip.ids.json")))["ids"]
overrides = R.apply(model, recipe, qdir, log=print)
before = np.array(qlab.logits_for(model, probe))
config = json.load(open(os.path.join(qlab.NATIVE, "config.json")))
quant = {"group_size": 64, "bits": 8, "mode": "affine"}
quant.update({k: overrides[k] for k in sorted(overrides)})
config["quantization"] = quant
config["quantization_config"] = quant
save_weights(out_dir, model)
save_config(config, os.path.join(out_dir, "config.json"))
for f in COPY:
shutil.copy2(os.path.join(qlab.NATIVE, f), os.path.join(out_dir, f))
os.makedirs(os.path.join(out_dir, "tiktoken"), exist_ok=True)
shutil.copy2(os.path.join(qlab.NATIVE, "tiktoken", "tokenizer.model"), os.path.join(out_dir, "tiktoken", "tokenizer.model"))
kinds = {}
for path, q in overrides.items():
group = "experts" if ".switch_mlp." in path else R.dense_group(path, None) or "other"
key = f"{group}: {q['mode']} {q['bits']}-bit g{q['group_size']}"
kinds[key] = kinds.get(key, 0) + 1
del model
mx.clear_cache()
reloaded, _ = load(out_dir)
after = np.array(qlab.logits_for(reloaded, probe))
max_diff = float(np.abs(before - after).max())
files = sorted(f for f in os.listdir(out_dir) if f.endswith(".safetensors"))
manifest = dict(
name=name, recipe=recipe, upstream="thinkingmachines/Inkling-2-Mini", upstream_revision=UPSTREAM_REVISION,
module_counts=kinds, reload_max_logit_diff=max_diff,
weights_bytes=sum(os.path.getsize(os.path.join(out_dir, f)) for f in files if f != "mtp.safetensors"),
files={f: dict(bytes=os.path.getsize(os.path.join(out_dir, f)), sha256=sha256(os.path.join(out_dir, f))) for f in files},
)
json.dump(manifest, open(os.path.join(out_dir, "build_manifest.json"), "w"), indent=1)
print(json.dumps({k: v for k, v in manifest.items() if k not in ("files", "recipe")}, indent=1))
if max_diff != 0.0:
raise SystemExit(f"reloaded checkpoint differs from the in-memory model (max logit diff {max_diff})")
if __name__ == "__main__":
main()
|