Image-Text-to-Text
MLX
Safetensors
English
inkling2_loop
mlx-vlm
inkling2
Mixture of Experts
quantized
gptq
apple-silicon
tool-calling
agents
conversational
8-bit precision
Instructions to use VisualInference/Inkling-2-Mini-MLX-2bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use VisualInference/Inkling-2-Mini-MLX-2bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("VisualInference/Inkling-2-Mini-MLX-2bit") config = load_config("VisualInference/Inkling-2-Mini-MLX-2bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use VisualInference/Inkling-2-Mini-MLX-2bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-2bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "VisualInference/Inkling-2-Mini-MLX-2bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use VisualInference/Inkling-2-Mini-MLX-2bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-2bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default VisualInference/Inkling-2-Mini-MLX-2bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use VisualInference/Inkling-2-Mini-MLX-2bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-2bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "VisualInference/Inkling-2-Mini-MLX-2bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download extras/scripts/recipes.py from VisualInference/Inkling-2-Mini-MLX-2bit: direct link, hf CLI and curl.
- Browser
- Download file 4.35 kB
-
https://huggingface.co/VisualInference/Inkling-2-Mini-MLX-2bit/resolve/main/extras/scripts/recipes.py
- Command line
-
hf download hf://VisualInference/Inkling-2-Mini-MLX-2bit/extras/scripts/recipes.py
-
curl -L -o recipes.py https://huggingface.co/VisualInference/Inkling-2-Mini-MLX-2bit/resolve/main/extras/scripts/recipes.py
4.35 kB
| """Apply a quantization recipe to the loaded native model (in memory), and describe it for config.json. | |
| A recipe is a dict: | |
| {"experts": {"default": SPEC, "layers": {"20": SPEC, ...}}, SPEC = "native" | "rtn:BITS:GROUP" | "gptq:BITS:GROUP" | |
| "dense": {"attn": Q, "embed": Q, "head": Q, "vision": Q, "draft": Q}} Q = null (keep float) | [BITS, GROUP] | |
| "gptq" shards are read from `qdir` (files q_LNN_bBgG.safetensors written by gptq.py). | |
| """ | |
| import os | |
| import mlx.core as mx | |
| import mlx.nn as nn | |
| import qlab | |
| ATTN_BIG = ("q_proj", "k_proj", "v_proj", "r_proj", "o_proj", "gate_proj") | |
| def expert_spec(recipe, layer): | |
| ex = recipe.get("experts", {}) | |
| return ex.get("layers", {}).get(str(layer), ex.get("default", "native")) | |
| def dense_group(path, module): | |
| """Which recipe group a dense module belongs to, or None if it must stay in floating point.""" | |
| if ".mlp.switch_mlp." in path: | |
| return None | |
| if path == "language_model.model.embed_tokens": | |
| return "embed" | |
| if path == "language_model.lm_head": | |
| return "head" | |
| if path.startswith("language_model.model.layers.") and ".self_attn." in path: | |
| leaf = path.rsplit(".", 1)[-1] | |
| # sink_proj (24 rows) and the dynamic-convolution tap projections are tiny and stay bf16. | |
| return "attn" if leaf in ATTN_BIG and ".k_sconv." not in path and ".v_sconv." not in path else None | |
| if path.startswith("vision_tower."): | |
| return "vision" | |
| return None | |
| def apply_experts(model, recipe, qdir=None, log=None): | |
| """Replace expert projections according to the recipe. Returns {module_path: quantization dict}.""" | |
| overrides = {} | |
| act_dtype = model.language_model.model.embed_norm.weight.dtype | |
| for li, layer in enumerate(qlab.layers(model)): | |
| spec = expert_spec(recipe, li) | |
| sw = layer.mlp.switch_mlp | |
| base = f"language_model.model.layers.{li}.mlp.switch_mlp." | |
| if spec == "native": | |
| for p in qlab.PROJECTIONS: | |
| proj = getattr(sw, p) | |
| overrides[base + p] = {"group_size": proj.group_size, "bits": proj.bits, "mode": proj.mode} | |
| continue | |
| kind, bits, group = spec.split(":") | |
| bits, group = int(bits), int(group) | |
| if kind == "gptq": | |
| shard = mx.load(os.path.join(qdir, f"q_L{li:02d}_b{bits}g{group}.safetensors")) | |
| for p in qlab.PROJECTIONS: | |
| if kind == "gptq": | |
| packed, s, b = shard[p + ".weight"], shard[p + ".scales"], shard[p + ".biases"] | |
| elif kind == "rtn": | |
| packed, s, b = qlab.rtn_affine(qlab.dense_expert(getattr(sw, p)), bits, group) | |
| else: | |
| raise ValueError(spec) | |
| # Scales and offsets are stored in the activation dtype. MLX's expert kernel (gather_qmm) promotes to | |
| # float32 and runs about three times slower when their dtype differs from the input's. | |
| s, b = s.astype(act_dtype), b.astype(act_dtype) | |
| mx.eval(packed, s, b) | |
| setattr(sw, p, qlab.affine_switch(packed, s, b, bits, group)) | |
| overrides[base + p] = {"group_size": group, "bits": bits, "mode": "affine"} | |
| mx.clear_cache() | |
| if log: | |
| log(f"layer {li}: experts -> {spec}") | |
| return overrides | |
| def apply_dense(model, recipe): | |
| """Affine-quantize dense groups. Returns {module_path: quantization dict}.""" | |
| dense = recipe.get("dense", {}) or {} | |
| overrides = {} | |
| def predicate(path, module): | |
| if not hasattr(module, "to_quantized") or not hasattr(module, "weight"): | |
| return False | |
| q = dense.get(dense_group(path, module)) | |
| if not q: | |
| return False | |
| bits, group = q | |
| if module.weight.shape[-1] % group != 0: | |
| return False | |
| overrides[path] = {"group_size": group, "bits": bits, "mode": "affine"} | |
| return {"group_size": group, "bits": bits, "mode": "affine"} | |
| nn.quantize(model, class_predicate=predicate) | |
| return overrides | |
| def apply(model, recipe, qdir=None, log=None): | |
| overrides = apply_experts(model, recipe, qdir, log) | |
| overrides.update(apply_dense(model, recipe)) | |
| mx.eval(model.parameters()) | |
| return overrides | |
| def nbytes(model): | |
| from mlx.utils import tree_flatten | |
| return sum(v.nbytes for _, v in tree_flatten(model.parameters())) | |