Image-Text-to-Text
MLX
Safetensors
English
inkling2_loop
mlx-vlm
inkling2
Mixture of Experts
quantized
gptq
apple-silicon
tool-calling
agents
conversational
8-bit precision
Instructions to use VisualInference/Inkling-2-Mini-MLX-3bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use VisualInference/Inkling-2-Mini-MLX-3bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("VisualInference/Inkling-2-Mini-MLX-3bit") config = load_config("VisualInference/Inkling-2-Mini-MLX-3bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use VisualInference/Inkling-2-Mini-MLX-3bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-3bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "VisualInference/Inkling-2-Mini-MLX-3bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use VisualInference/Inkling-2-Mini-MLX-3bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-3bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default VisualInference/Inkling-2-Mini-MLX-3bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use VisualInference/Inkling-2-Mini-MLX-3bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-3bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "VisualInference/Inkling-2-Mini-MLX-3bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
File size: 17,628 Bytes
e0b70af | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 | """Conversation scenarios for self-generated calibration / evaluation chats and agent fixtures."""
import json
def fn(name, description, properties, required):
return {
"type": "function",
"function": {
"name": name,
"description": description,
"parameters": {"type": "object", "properties": properties, "required": required},
},
}
TOOLS = {
"get_weather": fn(
"get_weather", "Get the current weather for a city.",
{"city": {"type": "string"}, "unit": {"type": "string", "enum": ["celsius", "fahrenheit"]}}, ["city"],
),
"terminal": fn(
"terminal", "Run a shell command in the project sandbox and return stdout, stderr and the exit code.",
{"command": {"type": "string", "description": "Command to run"}, "timeout": {"type": "integer"}}, ["command"],
),
"read_file": fn(
"read_file", "Read a text file and return its contents with line numbers.",
{"path": {"type": "string"}, "offset": {"type": "integer"}, "limit": {"type": "integer"}}, ["path"],
),
"write_file": fn(
"write_file", "Create or overwrite a text file.",
{"path": {"type": "string"}, "content": {"type": "string"}}, ["path", "content"],
),
"web_search": fn(
"web_search", "Search the web and return the top results.",
{"query": {"type": "string"}, "max_results": {"type": "integer"}}, ["query"],
),
"calculator": fn(
"calculator", "Evaluate an arithmetic expression.", {"expression": {"type": "string"}}, ["expression"]
),
"search_orders": fn(
"search_orders", "Look up customer orders.",
{
"customer_id": {"type": "string"},
"status": {"type": "string", "enum": ["open", "shipped", "returned"]},
"filters": {
"type": "object",
"properties": {"min_total": {"type": "number"}, "tags": {"type": "array", "items": {"type": "string"}}},
},
},
["customer_id"],
),
}
FILES = {
"app/config.py": 'import os\n\nDEBUG = os.environ.get("DEBUG", "0") == "1"\nTIMEOUT = 30\nRETRIES = 3\nDATABASE_URL = os.environ["DATABASE_URL"]\n',
"README.md": "# Tidepool\n\nA small job queue.\n\n## Run\n\n make dev\n\n## Test\n\n pytest -q\n",
"src/queue.py": (
"import heapq\n\n\nclass JobQueue:\n def __init__(self):\n self._heap = []\n self._count = 0\n\n"
" def push(self, job, priority=0):\n heapq.heappush(self._heap, (priority, self._count, job))\n self._count += 1\n\n"
" def pop(self):\n return heapq.heappop(self._heap)[1]\n\n def __len__(self):\n return len(self._heap)\n"
),
}
def _terminal(cmd):
"""A tiny scripted shell over FILES: enough for an agent to explore, run tests and commit."""
import re
out, err, code = [], [], 0
for part in re.split(r"\s*(?:&&|;|\|\|)\s*", cmd):
part = part.strip()
if not part:
continue
head = part.split("|")[0].strip()
words = head.split()
prog = words[0] if words else ""
if "pytest" in part:
fixed = "return heapq.heappop(self._heap)[2]" in FILES["src/queue.py"]
if fixed:
out.append(".. [100%]\n2 passed in 0.02s")
else:
code = 1
out.append("F. [100%]\n=== FAILURES ===\n___ test_pop_returns_job ___\n def test_pop_returns_job():\n q = JobQueue(); q.push('a')\n> assert q.pop() == 'a'\nE AssertionError: assert 0 == 'a'\ntests/test_queue.py:7: AssertionError\n1 failed, 1 passed in 0.03s")
elif prog == "ls":
target = next((w for w in words[1:] if not w.startswith("-")), ".").rstrip("/")
names = sorted({f.split("/")[0] if target in (".", "") else f[len(target) + 1:].split("/")[0]
for f in list(FILES) + ["tests/test_queue.py", "Makefile"]
if target in (".", "") or f.startswith(target + "/")})
out.append("\n".join(names) if names else "") if names else err.append(f"ls: {target}: No such file or directory")
elif prog in ("cat", "head", "tail", "sed", "nl"):
paths = [w for w in words[1:]
if not w.startswith("-") and not w.isdigit() and ">" not in w and "," not in w and w != "p"]
for path in paths:
path = path.lstrip("./")
if path in FILES:
out.append(FILES[path].rstrip("\n"))
elif path == "Makefile":
out.append("dev:\n\tpython -m app\n\ntest:\n\tpytest -q")
elif path == "tests/test_queue.py":
out.append("from src.queue import JobQueue\n\n\ndef test_len():\n q = JobQueue(); q.push('a'); assert len(q) == 1\n\n\ndef test_pop_returns_job():\n q = JobQueue(); q.push('a')\n assert q.pop() == 'a'")
else:
code = 1; err.append(f"{prog}: {path}: No such file or directory")
elif prog == "pwd":
out.append("/workspace/tidepool")
elif prog == "echo":
out.append(" ".join(words[1:]).strip("'\""))
elif prog == "find":
out.append("\n".join("./" + f for f in sorted(list(FILES) + ["tests/test_queue.py", "Makefile"])))
elif prog == "grep":
pat = next((w.strip("'\"") for w in words[1:] if not w.startswith("-")), "")
hits = [f"{f}:{i + 1}:{line}" for f, body in FILES.items() for i, line in enumerate(body.splitlines()) if pat and pat in line]
out.append("\n".join(hits)) if hits else None
code = 0 if hits else 1
elif prog == "git":
if "status" in part:
out.append("On branch main\nnothing to commit, working tree clean")
elif "diff" in part:
out.append("")
elif "log" in part:
out.append("a1b2c3d Add priority queue\n9f8e7d6 Initial commit")
else:
out.append("")
elif prog in ("python", "python3", "make"):
out.append("")
elif prog == "wc":
out.append(" 17 src/queue.py")
else:
code = 127; err.append(f"sh: {prog}: command not found")
return {"exit_code": code, "stdout": "\n".join(x for x in out if x), "stderr": "\n".join(err)}
def run_tool(name, args):
"""Deterministic canned tool results."""
if name == "get_weather":
table = {"lisbon": (19, "clear"), "oslo": (3, "snow"), "tokyo": (14, "rain"), "cairo": (28, "sunny")}
t, sky = table.get(str(args.get("city", "")).lower(), (12, "cloudy"))
if args.get("unit") == "fahrenheit":
return json.dumps({"city": args.get("city"), "temp_f": round(t * 9 / 5 + 32), "sky": sky})
return json.dumps({"city": args.get("city"), "temp_c": t, "sky": sky})
if name == "terminal":
return json.dumps(_terminal(str(args.get("command", ""))))
if name == "read_file":
body = FILES.get(str(args.get("path", "")))
if body is None:
return json.dumps({"error": f"No such file: {args.get('path')}"})
return "\n".join(f"{i + 1}\t{line}" for i, line in enumerate(body.splitlines()))
if name == "write_file":
path = str(args.get("path", "")).lstrip("./")
FILES[path] = str(args.get("content", ""))
return json.dumps({"ok": True, "bytes": len(FILES[path])})
if name == "web_search":
return json.dumps({"results": [
{"title": "MLX documentation", "url": "https://ml-explore.github.io/mlx/", "snippet": "MLX is an array framework for machine learning on Apple silicon."},
{"title": "Unified memory on Apple silicon", "url": "https://example.org/unified-memory", "snippet": "CPU and GPU share one memory pool, so arrays need no copies."},
]})
if name == "calculator":
try:
return json.dumps({"result": eval(str(args.get("expression", "0")), {"__builtins__": {}}, {})})
except Exception as exc: # noqa: BLE001
return json.dumps({"error": str(exc)})
if name == "search_orders":
return json.dumps({"orders": [
{"id": "A-1042", "status": "shipped", "total": 129.5, "tags": ["gift"]},
{"id": "A-1077", "status": "open", "total": 48.0, "tags": []},
]})
return json.dumps({"error": f"unknown tool {name}"})
AGENT_SYSTEM = (
"You are a coding agent working in a sandboxed repository. Use the tools to inspect and change files. "
"Keep answers short and report what you actually did."
)
ARTICLE = (
"The municipal archive reopened on Monday after a two-year renovation. The work replaced the roof, added a "
"climate-controlled vault for the oldest parish registers, and moved the reading room to the ground floor so "
"that visitors no longer need the narrow staircase. Opening hours are now 9:00 to 17:00 on weekdays and 10:00 "
"to 14:00 on Saturdays. Researchers must book a desk at least one day ahead; the archive holds 24 desks. "
"Digitisation continues: about 40 percent of the pre-1900 registers can be read online, and the team expects "
"to reach 60 percent by the end of next year. The renovation cost 3.2 million euros, of which the region paid half."
)
# (id, split, category, kwargs)
S = []
def add(sid, split, category, user, tools=None, system=None, effort="low", max_tokens=700, followups=()):
S.append(dict(id=sid, split=split, category=category, user=user, tools=tools or [], system=system,
effort=effort, max_tokens=max_tokens, followups=list(followups)))
# --- tool use (agent fixtures and calibration) ---
add("tool_weather_pair", "calib", "tools", "Is it warmer in Lisbon or in Oslo right now? Use the tool.", ["get_weather"])
add("tool_weather_single", "eval", "tools", "What's the weather in Tokyo in fahrenheit?", ["get_weather", "calculator"])
add("tool_none_needed", "calib", "tools", "Thanks! No lookup needed, just tell me a fun fact about lighthouses.", ["get_weather", "web_search"])
add("agent_fix_test", "calib", "agent", "The test suite is failing. Find out why and fix it.", ["terminal", "read_file", "write_file"], AGENT_SYSTEM, "medium", 900)
add("agent_explore", "eval", "agent", "What does this project do and how do I run its tests?", ["terminal", "read_file"], AGENT_SYSTEM, "low", 700)
add("agent_edit_config", "calib", "agent", "Change the request timeout to 60 seconds in the config and show me the diff in words.", ["terminal", "read_file", "write_file"], AGENT_SYSTEM, "low", 800)
add("agent_missing_file", "eval", "agent", "Open docs/ARCHITECTURE.md and summarise it.", ["read_file", "terminal"], AGENT_SYSTEM, "low", 500)
add("tool_nested_args", "calib", "tools", "Find customer C-88's shipped orders over 100 dollars that are tagged gift.", ["search_orders"], None, "low", 500)
add("tool_nested_args_eval", "eval", "tools", "List open orders for customer C-19 with a total of at least 40.", ["search_orders", "calculator"], None, "low", 500)
add("tool_search_then_answer", "calib", "tools", "Search for what MLX is and answer in two sentences with the source URL.", ["web_search"], None, "low", 500)
add("tool_calc_chain", "eval", "tools", "Use the calculator: what is (1234 * 57) - 999, and what is that divided by 7?", ["calculator"], None, "low", 500)
add("tool_multiturn", "calib", "tools", "What's the weather in Cairo?", ["get_weather"], None, "none", 400,
followups=["And in Oslo? Which one needs a coat?"])
# --- coding ---
add("code_lru", "calib", "code", "Write a small LRU cache class in Python with get and put, O(1) each, plus three assert-based tests.", max_tokens=800)
add("code_js_debounce", "calib", "code", "Write a debounce function in TypeScript with a cancel method and explain when leading-edge firing is useful.", max_tokens=700)
add("code_sql", "eval", "code", "Given tables orders(id, customer_id, total, created_at) and customers(id, country), write SQL for monthly revenue per country for 2025, and say which index helps.", max_tokens=700)
add("code_bash", "calib", "code", "Write a bash script that finds the ten largest files under a directory given as $1, safely handling spaces in names.", max_tokens=600)
add("code_review", "eval", "code", "Review this function and point out bugs:\n\n```python\ndef median(xs):\n xs.sort()\n n = len(xs)\n return xs[n/2] if n % 2 else (xs[n/2] + xs[n/2+1]) / 2\n```", max_tokens=700)
add("code_rust", "calib", "code", "Explain Rust's ownership rules with one short example that fails to compile and its fix.", max_tokens=700)
add("code_regex", "eval", "code", "Give a regular expression for ISO 8601 dates (YYYY-MM-DD) that rejects month 13 and day 32, with a brief explanation of each part.", max_tokens=600)
# --- reasoning / math ---
add("math_trains", "calib", "reasoning", "Two trains start 300 km apart and move toward each other at 70 km/h and 80 km/h. When do they meet, and how far has each travelled?", effort="medium")
add("math_prob", "eval", "reasoning", "I roll two fair dice. What is the probability that the product is even? Show the reasoning briefly.", effort="medium")
add("logic_puzzle", "calib", "reasoning", "Ana, Ben and Cy each own one pet: a cat, a dog, a fish. Ana does not own the dog. Ben owns neither the dog nor the fish. Who owns what?", effort="high", max_tokens=900)
add("estimate", "eval", "reasoning", "Estimate how many piano tuners work in a city of two million people. State your assumptions.", effort="medium")
add("plan_trip", "calib", "reasoning", "Plan a two-day itinerary for Lisbon for someone who likes architecture and dislikes crowds. Keep it under 200 words.", effort="low")
# --- summarisation / extraction / structured output ---
add("summ_article", "calib", "text", "Summarise in three bullet points:\n\n" + ARTICLE, effort="none")
add("extract_json", "calib", "structured", "Extract the facts from this text as JSON with keys opening_hours (object), desks (integer), cost_eur (number) and digitised_percent (integer). Output only the JSON.\n\n" + ARTICLE, effort="none", max_tokens=400)
add("extract_json_eval", "eval", "structured", "Return a JSON array of objects {\"name\", \"role\", \"year\"} for: Ada Lovelace wrote notes on the Analytical Engine in 1843; Grace Hopper built the A-0 compiler in 1952; Margaret Hamilton led Apollo flight software in 1969. Output only the JSON.", effort="none", max_tokens=400)
add("rewrite_tone", "eval", "text", "Rewrite this to be polite and concise: 'You sent the wrong file again. Fix it now, I am tired of waiting.'", effort="none", max_tokens=300)
add("classify", "calib", "structured", "Classify each review as positive, negative or mixed and answer as a markdown table:\n1. 'Battery lasts forever but the screen scratches easily.'\n2. 'Arrived broken.'\n3. 'Exactly what I needed.'", effort="none", max_tokens=400)
# --- knowledge / explanation ---
add("explain_tcp", "calib", "knowledge", "Explain the TCP three-way handshake to a new engineer in about 120 words.", effort="none", max_tokens=400)
add("explain_photosynth", "eval", "knowledge", "Explain photosynthesis at two levels: for a ten-year-old, then for a biology undergraduate.", effort="low", max_tokens=600)
add("compare", "calib", "knowledge", "Compare B-trees and LSM-trees for a write-heavy workload. Give a recommendation.", effort="low", max_tokens=600)
add("history", "eval", "knowledge", "Why did the Hanseatic League decline? Give the three main causes.", effort="low", max_tokens=500)
# --- multilingual ---
add("fr_email", "calib", "multilingual", "Rédige un court e-mail professionnel pour reporter une réunion à jeudi 15 h, en t'excusant du changement.", effort="none", max_tokens=350)
add("de_explain", "eval", "multilingual", "Erkläre in einfachem Deutsch, was eine Rekursion ist, mit einem Beispiel aus dem Alltag.", effort="none", max_tokens=350)
add("es_recipe", "calib", "multilingual", "Dame una receta sencilla de tortilla de patatas para dos personas, con cantidades.", effort="none", max_tokens=400)
add("ja_translate", "calib", "multilingual", "次の文を英語と中国語に翻訳してください:「明日の会議は午後三時からオンラインで行います。」", effort="none", max_tokens=300)
add("zh_summary", "eval", "multilingual", "用三句话解释什么是机器学习中的过拟合,并给出一个避免的方法。", effort="none", max_tokens=350)
add("pt_story", "calib", "multilingual", "Escreve uma história muito curta (80 palavras) sobre um farol e uma gaivota.", effort="none", max_tokens=350)
# --- system-prompted persona / format constraints ---
add("sys_terse", "calib", "system", "How do I undo my last git commit but keep the changes?", system="Answer in at most two sentences. Never use markdown.", effort="none", max_tokens=200)
add("sys_json_only", "eval", "system", "Book a table for four at 19:30 tomorrow under the name Okafor.", system='You are an intent parser. Reply with exactly one JSON object: {"intent": string, "slots": object}. No prose.', effort="none", max_tokens=250)
add("sys_refuse_scope", "calib", "system", "What's a good pasta recipe?", system="You are the support assistant for a tax-filing product. Only answer questions about the product; politely decline anything else.", effort="none", max_tokens=250)
SCENARIOS = S
|