Image-Text-to-Text
MLX
Safetensors
English
inkling2_loop
mlx-vlm
inkling2
Mixture of Experts
quantized
gptq
apple-silicon
tool-calling
agents
conversational
8-bit precision
Instructions to use VisualInference/Inkling-2-Mini-MLX-4bit with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- MLX
How to use VisualInference/Inkling-2-Mini-MLX-4bit with MLX:
# Make sure mlx-vlm is installed # pip install --upgrade mlx-vlm from mlx_vlm import load, generate from mlx_vlm.prompt_utils import apply_chat_template from mlx_vlm.utils import load_config # Load the model model, processor = load("VisualInference/Inkling-2-Mini-MLX-4bit") config = load_config("VisualInference/Inkling-2-Mini-MLX-4bit") # Prepare input image = ["http://images.cocodataset.org/val2017/000000039769.jpg"] prompt = "Describe this image." # Apply chat template formatted_prompt = apply_chat_template( processor, config, prompt, num_images=1 ) # Generate output output = generate(model, processor, formatted_prompt, image) print(output) - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- LM Studio
- Pi
How to use VisualInference/Inkling-2-Mini-MLX-4bit with Pi:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-4bit"
Configure the model in Pi
# Install Pi: npm install -g @earendil-works/pi-coding-agent # Add to ~/.pi/agent/models.json: { "providers": { "mlx-lm": { "baseUrl": "http://localhost:8080/v1", "api": "openai-completions", "apiKey": "none", "models": [ { "id": "VisualInference/Inkling-2-Mini-MLX-4bit" } ] } } }Run Pi
# Start Pi in your project directory: pi
- Hermes Agent
How to use VisualInference/Inkling-2-Mini-MLX-4bit with Hermes Agent:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-4bit"
Configure Hermes
# Install Hermes: curl -fsSL https://hermes-agent.nousresearch.com/install.sh | bash hermes setup # Point Hermes at the local server: hermes config set model.provider custom hermes config set model.base_url http://127.0.0.1:8080/v1 hermes config set model.default VisualInference/Inkling-2-Mini-MLX-4bit
Run Hermes
hermes
- Atomic Chat
- OpenClaw
How to use VisualInference/Inkling-2-Mini-MLX-4bit with OpenClaw:
Start the MLX server
# Install MLX LM: uv tool install mlx-lm # Start a local OpenAI-compatible server: mlx_lm.server --model "VisualInference/Inkling-2-Mini-MLX-4bit"
Configure OpenClaw
# Install OpenClaw: npm install -g openclaw@latest # Register the local server and set it as the default model: openclaw onboard --non-interactive --mode local \ --auth-choice custom-api-key \ --custom-base-url http://127.0.0.1:8080/v1 \ --custom-model-id "VisualInference/Inkling-2-Mini-MLX-4bit" \ --custom-provider-id mlx-lm \ --custom-compatibility openai \ --custom-text-input \ --accept-risk \ --skip-health
Run OpenClaw
openclaw agent --local --agent main --message "Hello from Hugging Face"
Download extras/mlx-vlm-inkling2-tool-calls.patch from VisualInference/Inkling-2-Mini-MLX-4bit: direct link, hf CLI and curl.
- Browser
- Download file 49.4 kB
-
https://huggingface.co/VisualInference/Inkling-2-Mini-MLX-4bit/resolve/main/extras/mlx-vlm-inkling2-tool-calls.patch
- Command line
-
hf download hf://VisualInference/Inkling-2-Mini-MLX-4bit/extras/mlx-vlm-inkling2-tool-calls.patch
-
curl -L -o mlx-vlm-inkling2-tool-calls.patch https://huggingface.co/VisualInference/Inkling-2-Mini-MLX-4bit/resolve/main/extras/mlx-vlm-inkling2-tool-calls.patch
49.4 kB
diff --git a/mlx_vlm/server/anthropic.py b/mlx_vlm/server/anthropic.py
index ed46ff05..1b714bec 100644
--- a/mlx_vlm/server/anthropic.py
+++ b/mlx_vlm/server/anthropic.py
@@ -29,11 +29,11 @@ from .generation import (
from .responses_state import (
ThinkingStreamState,
ToolCallStreamState,
+ content_beside_tool_calls,
finish_content_streams,
make_response_stream_state,
parse_assistant_tool_calls,
prompt_has_open_thinking,
- strip_protocol_markers,
)
from .runtime import runtime
from .schemas import AnthropicMessageResponse, AnthropicRequest, AnthropicUsage
@@ -1130,20 +1130,10 @@ async def anthropic_messages_endpoint(http_request: Request):
tc = parse_tools(full_text)
if tc.calls:
parsed_tool_calls = tc.calls
- # Structured templates already removed tool/routing regions
- # from the content parsed above. Otherwise use the delimiter
- # parser's remainder (reasoning has already been excluded).
- if not (
- getattr(processor, "response_template", None)
- or getattr(
- getattr(processor, "tokenizer", None),
- "response_template",
- None,
- )
- ):
- content = tc.remaining_text or ""
- content = strip_protocol_markers(
+ content = content_beside_tool_calls(
content,
+ tc.remaining_text,
+ processor,
tool_module,
gen_args.thinking_start_token,
gen_args.thinking_end_token,
diff --git a/mlx_vlm/server/openai.py b/mlx_vlm/server/openai.py
index a1729e13..3d4d0048 100644
--- a/mlx_vlm/server/openai.py
+++ b/mlx_vlm/server/openai.py
@@ -60,13 +60,13 @@ from .responses_state import (
from .responses_state import _sse_event as _response_sse_event
from .responses_state import (
_store_response,
+ content_beside_tool_calls,
finish_content_streams,
make_response_stream_state,
parse_assistant_tool_calls,
prompt_has_open_thinking,
response_store,
response_store_lock,
- strip_protocol_markers,
)
from .runtime import runtime
from .schemas import (
@@ -2807,11 +2807,11 @@ async def chat_completions_endpoint(request: ChatRequest, http_request: Request)
tc = parse_tools(full_text)
if tc.calls:
parsed_tool_calls = tc.calls
- # Reasoning was already removed before tool parsing.
- clean_remaining = tc.remaining_text or ""
content = (
- strip_protocol_markers(
- clean_remaining,
+ content_beside_tool_calls(
+ content,
+ tc.remaining_text,
+ processor,
tool_module,
gen_args.thinking_start_token,
gen_args.thinking_end_token,
diff --git a/mlx_vlm/server/responses_state.py b/mlx_vlm/server/responses_state.py
index d975a822..424a54ea 100644
--- a/mlx_vlm/server/responses_state.py
+++ b/mlx_vlm/server/responses_state.py
@@ -9,6 +9,7 @@ from threading import Lock
from typing import Any, Dict, List, Optional, Tuple
from fastapi import HTTPException
+from transformers.utils.chat_parsing import ResponseParser, parse_response
from transformers.utils.chat_parsing.response_templates import load_response_template
from ..prompt_utils import _normalize_tool_message
@@ -241,6 +242,29 @@ def _response_template_prefix(tokenizer, prompt: str) -> str:
return prompt if template.start_anchor_re.search(prompt) else ""
+_TEXT_FIELDS = ("content", "reasoning", "reasoning_content")
+_REGION_KEYS = ("open", "open_pattern", "close", "close_pattern", "repeats")
+
+
+def _text_response_template(template: Dict[str, Any]) -> Dict[str, Any]:
+ """Copy a response template, reading its structured fields as plain text.
+
+ The server takes reasoning and content from a response template; tool calls
+ come from the tool parser. A structured field such as a JSON tool call is
+ decoded when its region closes, and a malformed or truncated one raises
+ there, which would lose the reasoning and content around it. Those fields
+ keep their delimiters, so their text still stays out of the content.
+ """
+ fields = {}
+ for name, field in template["fields"].items():
+ if name not in _TEXT_FIELDS:
+ field = {key: field[key] for key in _REGION_KEYS if key in field}
+ # A transform is what lets an open pattern keep its named groups.
+ field["transform"] = "{content}"
+ fields[name] = field
+ return {**template, "fields": fields}
+
+
def make_response_stream_state(
processor,
enable_thinking: bool = False,
@@ -253,8 +277,9 @@ def make_response_stream_state(
if tokenizer is not None and hasattr(tokenizer, "get_response_parser"):
try:
return ResponseTemplateStreamState(
- tokenizer.get_response_parser(
- prefix=_response_template_prefix(tokenizer, prompt)
+ ResponseParser(
+ _text_response_template(tokenizer.response_template),
+ prefix=_response_template_prefix(tokenizer, prompt),
)
)
except (AttributeError, TypeError, ValueError):
@@ -438,6 +463,28 @@ def strip_protocol_markers(
return text.strip()
+def content_beside_tool_calls(
+ content: Optional[str],
+ remaining_text: Optional[str],
+ processor: Any,
+ tool_module: Any,
+ thinking_start_token: Optional[str] = None,
+ thinking_end_token: Optional[str] = None,
+) -> str:
+ """Return the visible content of a turn that also made tool calls.
+
+ ``content`` is what ``_split_thinking`` returned for the turn. A response
+ template already kept tool calls and message routing out of it. Without a
+ template, the content is ``remaining_text``, the tool parser's remainder
+ (reasoning was excluded before tool parsing).
+ """
+ if _response_template_tokenizer(processor) is None:
+ content = remaining_text
+ return strip_protocol_markers(
+ content or "", tool_module, thinking_start_token, thinking_end_token
+ )
+
+
def finish_content_streams(thinking_state, tool_call_state):
"""``(reasoning, content)`` still held when a stream ends.
@@ -494,13 +541,23 @@ def _split_thinking(
tokenizer = _response_template_tokenizer(processor)
if tokenizer is not None and hasattr(tokenizer, "parse_response"):
try:
- parsed = tokenizer.parse_response(
- text, prefix=_response_template_prefix(tokenizer, prompt)
- )
+ prefix = _response_template_prefix(tokenizer, prompt)
+ try:
+ parsed = tokenizer.parse_response(text, prefix=prefix)
+ except ValueError:
+ # A structured field, such as a truncated tool call, did not
+ # decode. The reasoning and content around it are still valid.
+ parsed = parse_response(
+ text,
+ _text_response_template(tokenizer.response_template),
+ prefix=prefix,
+ )
+ # A turn of tool calls alone is a parsed turn with no content.
if isinstance(parsed, dict) and (
"content" in parsed
or "reasoning" in parsed
or "reasoning_content" in parsed
+ or "tool_calls" in parsed
):
reasoning = parsed.get("reasoning_content") or parsed.get("reasoning")
content = parsed.get("content")
@@ -556,7 +613,10 @@ def parse_assistant_tool_calls(
Response-template parsers may consume tool regions, so split reasoning with
the delimiter state instead. Examples inside reasoning are never tool calls.
+ A parser with ``find_tool_calls`` locates calls in the whole turn itself.
"""
+ if hasattr(tool_module, "find_tool_calls"):
+ return process_tool_calls(text or "", tool_module, tools)
content = (
ThinkingStreamState(
starts_in_thinking, thinking_start_token, thinking_end_token
@@ -606,8 +666,10 @@ def _response_output_items_from_text(
items = [
_tool_call_to_response_item(call, tool_registry) for call in tc.calls
]
- remaining = strip_protocol_markers(
- tc.remaining_text or "",
+ remaining = content_beside_tool_calls(
+ content,
+ tc.remaining_text,
+ processor,
tool_module,
thinking_start_token,
thinking_end_token,
diff --git a/mlx_vlm/tests/test_processors.py b/mlx_vlm/tests/test_processors.py
index e325391c..8f4cb6ad 100644
--- a/mlx_vlm/tests/test_processors.py
+++ b/mlx_vlm/tests/test_processors.py
@@ -2972,6 +2972,8 @@ WIRE_CALLS = {
"mistral": '[TOOL_CALLS]get_weather[ARGS]{"city": "Paris", "days": 3}',
"harmony": "<|channel|>commentary to=functions.get_weather <|constrain|>json"
'<|message|>{"city": "Paris", "days": 3}<|call|>',
+ "inkling2": "<|message_model|>get_weather<|content_invoke_tool_json|>"
+ '{"name":"get_weather","args":{"city":"Paris","days":3}}<|end_message|>',
"pythonic": '<|tool_call_start|>[get_weather(city="Paris", days=3)]<|tool_call_end|>',
"qwen3_coder": "<tool_call>\n<function=get_weather><parameter=city>Paris</parameter>"
"<parameter=days>3</parameter></function></tool_call>",
@@ -3162,6 +3164,9 @@ def test_deepseek_streamed_tool_calls(chunk_size):
("minicpm5", '<function name="lookup"><param>3</param></function>', None),
("gemma4", "just a normal model response, no tool call here", None),
("mistral", "not a tool call at all", None),
+ ("inkling2", '{"name":"get_weather","args":{"city":"Par', None),
+ ("inkling2", '{"args":{"city":"Paris"}}', "object with a name"),
+ ("inkling2", '{"name":"get_weather","args":["Paris"]}', "must be an object"),
(
"pythonic",
"[write_file(content='const player = { x: 0, y: 1 };)]",
@@ -3527,6 +3532,130 @@ def test_harmony_ignores_plain_commentary_preamble():
assert result.remaining_text == preamble
+def _inkling2_call(name, raw=None, **arguments):
+ body = raw or json.dumps(
+ dict(name=name, args=arguments), ensure_ascii=False, separators=(",", ":")
+ )
+ return f"<|message_model|>{name}<|content_invoke_tool_json|>{body}<|end_message|>"
+
+
+def _inkling2_calls(text):
+ result = process_tool_calls(text, load_tool_module("inkling2"), None)
+ calls = [
+ (call["function"]["name"], json.loads(call["function"]["arguments"]))
+ for call in result.calls
+ ]
+ return calls, result
+
+
+INKLING2_THINKING = (
+ "<|message_model|><|content_thinking|>I need both cities.<|end_message|>"
+)
+INKLING2_TEXT = "<|message_model|><|content_text|>Lisbon is warmer.<|end_message|>"
+INKLING2_ARGUMENTS = {
+ "nested": {"list": [1, 2.5, None, True, {"deep": []}], "empty": {}},
+ "unicode": "19°C — ação 日本 🐦",
+ "quotes": 'say "hi" \\ back\\slash',
+ "lines": "first\nsecond\ttabbed\r\n",
+ "markers": "<|end_message|><|message_model|>x<|content_invoke_tool_json|>{}",
+ "tags": "<think> </tool_call>",
+}
+
+
+def test_inkling2_parallel_calls_follow_reasoning():
+ text = (
+ INKLING2_THINKING
+ + _inkling2_call("get_weather", city="Lisbon", unit="celsius")
+ + _inkling2_call("get_weather", city="Oslo", unit="celsius")
+ )
+ calls, result = _inkling2_calls(text)
+ assert calls == [
+ ("get_weather", {"city": "Lisbon", "unit": "celsius"}),
+ ("get_weather", {"city": "Oslo", "unit": "celsius"}),
+ ]
+ assert [call["index"] for call in result.calls] == [0, 1]
+ assert len({call["id"] for call in result.calls}) == 2
+ # The role header and the tool name it carries leave with the call.
+ assert result.remaining_text == INKLING2_THINKING
+
+
+@pytest.mark.parametrize(
+ "text,expected",
+ [
+ (INKLING2_THINKING + INKLING2_TEXT, []),
+ (INKLING2_TEXT + _inkling2_call("get_time"), [("get_time", {})]),
+ (
+ "<|content_invoke_tool_json|>" + '{"name":"get_time"}' + "<|end_message|>",
+ [("get_time", {})],
+ ),
+ (
+ # Call markup quoted inside reasoning or text is not a call.
+ "<|message_model|><|content_thinking|>Not "
+ + _inkling2_call("delete_all")
+ + INKLING2_TEXT,
+ [],
+ ),
+ (
+ "<|message_model|><|content_text|>Example: <|content_invoke_tool_json|>"
+ '{"name":"delete_all","args":{}}<|end_message|>'
+ + _inkling2_call("get_time"),
+ [("get_time", {})],
+ ),
+ ],
+ ids=["text-only", "text-then-call", "bare-call", "in-reasoning", "in-text"],
+)
+def test_inkling2_calls_are_whole_messages(text, expected):
+ assert _inkling2_calls(text)[0] == expected
+
+
+@pytest.mark.parametrize("ascii_only", [False, True])
+def test_inkling2_arguments_round_trip(ascii_only):
+ body = json.dumps(
+ dict(name="write_file", args=INKLING2_ARGUMENTS), ensure_ascii=ascii_only
+ )
+ text = (
+ _inkling2_call("write_file", raw=body)
+ + INKLING2_TEXT
+ + _inkling2_call("get_time")
+ )
+ calls, result = _inkling2_calls(text)
+ assert calls == [("write_file", INKLING2_ARGUMENTS), ("get_time", {})]
+ assert result.remaining_text == INKLING2_TEXT
+ # Non-ASCII arguments are returned as written, not as escapes.
+ assert "19°C — ação 日本 🐦" in result.calls[0]["function"]["arguments"]
+
+
+@pytest.mark.parametrize(
+ "broken",
+ [
+ '{"name":"get_weather","args":{city}}',
+ '{"name":"get_weather","args":{"city":"unterminated}}',
+ '{"name":"get_weather","args":"Paris"}',
+ '["get_weather"]',
+ "",
+ ],
+)
+def test_inkling2_malformed_call_is_dropped_alone(broken):
+ text = (
+ _inkling2_call("get_time")
+ + _inkling2_call("get_weather", raw=broken or " ")
+ + INKLING2_TEXT
+ + _inkling2_call("get_date")
+ )
+ calls, result = _inkling2_calls(text)
+ assert calls == [("get_time", {}), ("get_date", {})]
+ assert result.remaining_text == INKLING2_TEXT
+
+
+@pytest.mark.parametrize("cut", range(1, 60, 3))
+def test_inkling2_truncated_call_is_not_a_call(cut):
+ complete = INKLING2_THINKING + _inkling2_call("get_time")
+ call = _inkling2_call("get_weather", city="Lisbon")
+ calls, result = _inkling2_calls(complete + call[:-cut])
+ assert calls == [("get_time", {})]
+ assert "get_time" not in result.remaining_text
+
+
# Loading and utility contracts
diff --git a/mlx_vlm/tests/test_server.py b/mlx_vlm/tests/test_server.py
index 59eb1e94..6e44662b 100644
--- a/mlx_vlm/tests/test_server.py
+++ b/mlx_vlm/tests/test_server.py
@@ -73,7 +73,14 @@ from mlx_vlm.server.responses_state import (
strip_protocol_markers,
)
from mlx_vlm.server.runtime_config import RuntimeConfig
-from mlx_vlm.tests.test_processors import MINICPM_MULTICALL
+from mlx_vlm.tests.test_processors import (
+ INKLING2_ARGUMENTS,
+ INKLING2_TEXT,
+ INKLING2_THINKING,
+ MINICPM_MULTICALL,
+ _inkling2_call,
+ fast_tokenizer,
+)
from mlx_vlm.tokenizer_utils import SPMStreamingDetokenizer, _ServerTokenStreamer
from mlx_vlm.tools import load_tool_module, process_tool_calls
@@ -3680,6 +3687,631 @@ def test_response_template_tool_calls(client, api):
assert "to=self" not in response.text and "<atem:" not in response.text
+@pytest.mark.parametrize("api", ["chat", "responses"])
+def test_response_template_content_excludes_tool_routing(client, api):
+ # Non-streamed, like the stream: a response template already kept the
+ # reasoning and the call's routing header out of the content.
+ config = NS(model_type="muse_glimmer")
+ tool = (
+ dict(type="function", name="get_weather", parameters={"type": "object"})
+ if api == "responses"
+ else _tool()
+ )
+ with _endpoint(
+ config=config,
+ processor=NS(config=config, tokenizer=_MuseResponseTemplateTokenizer()),
+ result=_result(_MUSE_CALL),
+ parser=load_tool_module("atem"),
+ ):
+ response = _post(client, api, tools=[tool])
+ body = response.json()
+ if api == "chat":
+ message = body["choices"][0]["message"]
+ assert message["content"] is None
+ assert message["reasoning_content"] == "I need the weather tool."
+ assert message["tool_calls"][0]["function"]["name"] == "get_weather"
+ else:
+ assert body["output_text"] == ""
+ assert [item["type"] for item in body["output"]] == [
+ "reasoning",
+ "function_call",
+ ]
+ assert "to=" not in response.text
+
+
+# Inkling 2 tool calls. The tokenizer is a real one carrying the response
+# template the model ships and a chat template condensed from the model's: the
+# same message framing, and the same refusal of string tool-call arguments.
+
+_INKLING2_OPEN = r"(?:<\|message_model\|>[^<]*)?<\|content_%s\|>"
+_INKLING2_RESPONSE_TEMPLATE = {
+ "defaults": {"role": "assistant"},
+ "start_anchor_pattern": r"<\|end_message\|>|<\|content_model_end_sampling\|>",
+ "fields": {
+ "reasoning_content": {
+ "open_pattern": _INKLING2_OPEN % "thinking",
+ "close": "<|end_message|>",
+ "repeats": True,
+ "join": "",
+ "content": "text",
+ "content_args": {"strip": False},
+ },
+ "content": {
+ "open_pattern": _INKLING2_OPEN % "(?:text|tool_error|invoke_tool_text)",
+ "close": "<|end_message|>",
+ "repeats": True,
+ "join": "",
+ "content": "text",
+ "content_args": {"strip": False},
+ },
+ "tool_calls": {
+ "open_pattern": _INKLING2_OPEN % "invoke_tool_json",
+ "close": "<|end_message|>",
+ "repeats": True,
+ "content": "json",
+ "transform": {
+ "type": "function",
+ "function": {"name": "{content.name}", "arguments": "{content.args}"},
+ },
+ },
+ },
+}
+_INKLING2_CHAT_TEMPLATE = """
+{%- set levels = {"none": "0", "minimal": "0.1", "low": "0.2", "medium": "0.7",
+ "high": "0.9", "xhigh": "0.99", "max": "0.99"} -%}
+{%- set roles = {"user": "<|message_user|>", "assistant": "<|message_model|>",
+ "system": "<|message_system|>", "tool": "<|message_tool|>"} -%}
+{%- if tools -%}
+ {{- "<|message_system|>tool_declare<|content_xml|>" ~ (tools | tojson) -}}
+ {{- "<|end_message|>" -}}
+{%- endif -%}
+{%- set effort = reasoning_effort if reasoning_effort is defined
+ and reasoning_effort is not none else "high" -%}
+{%- if effort not in levels -%}
+ {{- raise_exception("Unknown reasoning_effort: " ~ effort) -}}
+{%- endif -%}
+{{- "<|message_system|><|content_text|>Thinking effort level: " ~ levels[effort] -}}
+{{- "<|end_message|>" -}}
+{%- for message in messages -%}
+ {%- set header = namespace(text=roles[message.role]) -%}
+ {%- if message.role == "tool" -%}
+ {%- for turn in messages if turn.role == "assistant" and turn.tool_calls -%}
+ {%- for call in turn.tool_calls if call.id == message.tool_call_id -%}
+ {%- set header.text = header.text ~ call.function.name -%}
+ {%- endfor -%}
+ {%- endfor -%}
+ {%- endif -%}
+ {%- if message.role == "assistant" and message.reasoning_content -%}
+ {{- header.text ~ "<|content_thinking|>" ~ message.reasoning_content -}}
+ {{- "<|end_message|>" -}}
+ {%- endif -%}
+ {%- if message.content is string -%}
+ {%- if message.content -%}
+ {{- header.text ~ "<|content_text|>" ~ message.content -}}
+ {{- "<|end_message|>" -}}
+ {%- endif -%}
+ {%- else -%}
+ {%- for part in message.content or [] -%}
+ {{- header.text ~ "<|content_text|>" ~ part.text ~ "<|end_message|>" -}}
+ {%- endfor -%}
+ {%- endif -%}
+ {%- for call in message.tool_calls or [] -%}
+ {%- if call.function.arguments is string -%}
+ {{- raise_exception("tool call arguments must be a parsed object") -}}
+ {%- endif -%}
+ {{- "<|message_model|>" ~ call.function.name -}}
+ {{- '<|content_invoke_tool_json|>{"name":' ~ (call.function.name | tojson) -}}
+ {{- ',"args":' ~ (call.function.arguments | tojson(separators=(",", ":"))) -}}
+ {{- "}<|end_message|>" -}}
+ {%- endfor -%}
+ {%- if message.role == "assistant" -%}
+ {{- "<|content_model_end_sampling|>" -}}
+ {%- endif -%}
+{%- endfor -%}
+"""
+_INKLING2_LISBON = dict(city="Lisbon", unit="celsius")
+_INKLING2_OSLO = dict(city="Oslo", unit="celsius")
+_INKLING2_CALLS = _inkling2_call("get_weather", **_INKLING2_LISBON) + _inkling2_call(
+ "get_weather", **_INKLING2_OSLO
+)
+_INKLING2_WEATHER_CALLS = [
+ ("get_weather", _INKLING2_LISBON),
+ ("get_weather", _INKLING2_OSLO),
+]
+
+
+def _inkling2_request(
+ client,
+ text,
+ api="chat",
+ *,
+ stream=False,
+ chunk=0,
+ batched=True,
+ finish_reason="stop",
+ **payload,
+):
+ """Answer one request with the raw model output ``text``.
+
+ ``chunk`` is the size of the streamed pieces, so 1 splits every control
+ token and every JSON value. The finish reason arrives on a final empty
+ token, as it does after the end-of-turn token. Returns the response and
+ the prompts rendered by the real ``apply_chat_template``.
+ """
+ size = chunk or len(text)
+ pieces = [text[i : i + size] for i in range(0, len(text), size)]
+ tokens = [*map(_token, pieces), _token("", finish_reason=finish_reason)]
+ tokenizer = fast_tokenizer(
+ chat_template=_INKLING2_CHAT_TEMPLATE,
+ response_template=_INKLING2_RESPONSE_TEMPLATE,
+ )
+ prompts = []
+
+ def render(*args, **kwargs):
+ prompts.append(apply_chat_template(*args, **kwargs))
+ return prompts[-1]
+
+ with _endpoint(
+ config=NS(model_type="inkling2"),
+ processor=NS(tokenizer=tokenizer),
+ result=_result(text, finish_reason=finish_reason),
+ chunks=[_result(t.text, finish_reason=t.finish_reason) for t in tokens],
+ generator=_streaming(tokens) if batched else None,
+ ) as fake:
+ fake.template.side_effect = render
+ response = _post(client, api, stream=stream, **payload)
+ assert response.status_code == 200, response.text
+ return response, prompts
+
+
+def _inkling2_chat(response, stream):
+ """Reduce a chat completion, streamed or not, to one comparable turn."""
+ if stream:
+ events = _data(response)
+ assert not any("error" in event for event in events)
+ choices = [event["choices"][0] for event in events if event.get("choices")]
+ deltas = [choice["delta"] for choice in choices]
+ else:
+ choices = response.json()["choices"]
+ deltas = [choices[0]["message"]]
+ calls = {}
+ for delta in deltas:
+ for call in delta.get("tool_calls") or []:
+ function = call["function"]
+ entry = calls.setdefault(call["index"], dict(name="", arguments=""))
+ entry.update(id=call.get("id", entry.get("id")))
+ entry["name"] += function.get("name") or ""
+ entry["arguments"] += function.get("arguments") or ""
+ assert list(calls) == list(range(len(calls)))
+ return NS(
+ reasoning=_joined(deltas, "reasoning_content"),
+ content=_joined(deltas, "content"),
+ calls=[(c["name"], json.loads(c["arguments"])) for c in calls.values()],
+ ids=[c["id"] for c in calls.values()],
+ finish=[c["finish_reason"] for c in choices if c.get("finish_reason")],
+ deltas=deltas,
+ )
+
+
+@pytest.mark.parametrize(
+ "stream,chunk,batched",
+ [
+ (False, 0, True),
+ (False, 0, False),
+ (True, 0, True),
+ (True, 1, True),
+ (True, 1, False),
+ (True, 5, True),
+ (True, 11, False),
+ ],
+)
+def test_inkling2_chat_parallel_tool_calls(client, stream, chunk, batched):
+ response, _ = _inkling2_request(
+ client,
+ INKLING2_THINKING + _INKLING2_CALLS,
+ stream=stream,
+ chunk=chunk,
+ batched=batched,
+ tools=[_tool()],
+ )
+ turn = _inkling2_chat(response, stream)
+ assert turn.calls == _INKLING2_WEATHER_CALLS
+ assert len(set(turn.ids)) == 2 and all(turn.ids)
+ assert turn.finish == ["tool_calls"]
+ assert turn.reasoning == "I need both cities."
+ assert turn.content == ""
+ # No control token, and no tool name outside the calls themselves.
+ assert "<|" not in response.text
+ assert response.text.count("get_weather") == 2
+ if stream:
+ kinds = [
+ "reasoning" if delta.get("reasoning_content") else "call"
+ for delta in turn.deltas
+ if delta.get("reasoning_content") or delta.get("tool_calls")
+ ]
+ assert kinds == sorted(kinds, reverse=True) and set(kinds) == {
+ "reasoning",
+ "call",
+ }
+ else:
+ message = response.json()["choices"][0]["message"]
+ assert message["content"] is None
+ assert all(
+ isinstance(call["function"]["arguments"], str)
+ for call in message["tool_calls"]
+ )
+
+
+@pytest.mark.parametrize("stream,chunk", [(False, 0), (True, 1), (True, 7)])
+@pytest.mark.parametrize(
+ "text,reasoning,content,calls",
+ [
+ (_inkling2_call("get_time"), "", "", [("get_time", {})]),
+ (
+ INKLING2_THINKING + INKLING2_TEXT + _inkling2_call("get_time"),
+ "I need both cities.",
+ "Lisbon is warmer.",
+ [("get_time", {})],
+ ),
+ (
+ "<|message_model|><|content_text|>One. <|end_message|>"
+ + _inkling2_call("get_time")
+ + "<|message_model|><|content_tool_error|>Two.<|end_message|>",
+ "",
+ "One. Two.",
+ [("get_time", {})],
+ ),
+ (
+ INKLING2_THINKING + INKLING2_TEXT,
+ "I need both cities.",
+ "Lisbon is warmer.",
+ [],
+ ),
+ ],
+ ids=["call-only", "text-then-call", "text-around-call", "answer-only"],
+)
+def test_inkling2_chat_turn_shapes(
+ client, stream, chunk, text, reasoning, content, calls
+):
+ def turn(**payload):
+ response, _ = _inkling2_request(
+ client, text, stream=stream, chunk=chunk, **payload
+ )
+ assert "<|" not in response.text
+ return _inkling2_chat(response, stream)
+
+ with_tools = turn(tools=[_tool("get_time")])
+ assert with_tools.calls == calls
+ assert with_tools.reasoning == reasoning
+ assert with_tools.content == content
+ assert with_tools.finish == ["tool_calls" if calls else "stop"]
+ # Without tools nothing is executable; reasoning and content are the same.
+ without_tools = turn()
+ assert without_tools.calls == []
+ assert without_tools.finish == ["stop"]
+ assert without_tools.reasoning == reasoning
+ assert without_tools.content == content
+
+
+@pytest.mark.parametrize("stream,chunk", [(False, 0), (True, 1), (True, 9)])
+def test_inkling2_chat_arguments_round_trip(client, stream, chunk):
+ text = INKLING2_THINKING + _inkling2_call("write_file", **INKLING2_ARGUMENTS)
+ response, _ = _inkling2_request(
+ client, text, stream=stream, chunk=chunk, tools=[_tool("write_file")]
+ )
+ turn = _inkling2_chat(response, stream)
+ assert turn.calls == [("write_file", INKLING2_ARGUMENTS)]
+ assert turn.finish == ["tool_calls"]
+ assert turn.reasoning == "I need both cities."
+ assert turn.content == ""
+
+
+@pytest.mark.parametrize("stream,chunk", [(False, 0), (True, 1), (True, 6)])
+@pytest.mark.parametrize("batched", [False, True])
+@pytest.mark.parametrize(
+ "text,finish_reason,calls,finish",
+ [
+ # Cut by max_tokens inside the arguments: there is no call to run.
+ (
+ INKLING2_THINKING + _inkling2_call("get_weather", city="Lisbon")[:-24],
+ "length",
+ [],
+ "length",
+ ),
+ # A call completed before the cut is still returned.
+ (
+ _inkling2_call("get_time")
+ + _inkling2_call("get_weather", city="Lisbon")[:-24],
+ "length",
+ [("get_time", {})],
+ "tool_calls",
+ ),
+ (
+ _inkling2_call("get_weather", raw='{"name":"get_weather","args":{city}}'),
+ "stop",
+ [],
+ "stop",
+ ),
+ (
+ _inkling2_call("get_weather", raw='{"name":"get_weather"')
+ + _inkling2_call("get_time"),
+ "stop",
+ [("get_time", {})],
+ "tool_calls",
+ ),
+ ],
+ ids=["truncated", "truncated-after-call", "malformed", "malformed-then-call"],
+)
+def test_inkling2_chat_incomplete_calls(
+ client, stream, chunk, batched, text, finish_reason, calls, finish
+):
+ response, _ = _inkling2_request(
+ client,
+ text,
+ stream=stream,
+ chunk=chunk,
+ batched=batched,
+ finish_reason=finish_reason,
+ tools=[_tool("get_weather"), _tool("get_time")],
+ )
+ turn = _inkling2_chat(response, stream)
+ assert turn.calls == calls
+ assert turn.finish == [finish]
+ assert turn.content == ""
+ assert turn.reasoning == ("I need both cities." if "thinking" in text else "")
+ assert "<|" not in response.text and "city" not in response.text
+
+
+@pytest.mark.parametrize("stream,chunk", [(False, 0), (True, 0), (True, 1)])
+def test_inkling2_messages_tool_use(client, stream, chunk):
+ response, _ = _inkling2_request(
+ client,
+ INKLING2_THINKING + INKLING2_TEXT + _INKLING2_CALLS,
+ "messages",
+ stream=stream,
+ chunk=chunk,
+ tools=[_tool(api="messages")],
+ thinking=dict(type="enabled", budget_tokens=64),
+ max_tokens=256,
+ )
+ if stream:
+ events = _data(response)
+ blocks = [e["content_block"] for e in events if "content_block" in e]
+ inputs = [
+ e["delta"]["partial_json"]
+ for e in events
+ if e.get("delta", {}).get("type") == "input_json_delta"
+ ]
+ names = [block["name"] for block in blocks if block["type"] == "tool_use"]
+ assert len(names) == len(inputs)
+ calls = list(zip(names, map(json.loads, inputs)))
+ thinking, text = _thinking_text(response, "messages")
+ stop_reason = _anthropic_stream_stop(response)[0]
+ else:
+ body = response.json()
+ blocks = body["content"]
+ calls = [(b["name"], b["input"]) for b in blocks if b["type"] == "tool_use"]
+ thinking = _joined(blocks, "thinking")
+ text = _joined(blocks, "text")
+ stop_reason = body["stop_reason"]
+ assert [block["type"] for block in blocks] == [
+ "thinking",
+ "text",
+ "tool_use",
+ "tool_use",
+ ]
+ assert calls == _INKLING2_WEATHER_CALLS
+ assert len({b["id"] for b in blocks if b["type"] == "tool_use"}) == 2
+ assert thinking == "I need both cities."
+ assert text == "Lisbon is warmer."
+ assert stop_reason == "tool_use"
+ assert "<|" not in response.text
+
+
+@pytest.mark.parametrize("stream,chunk", [(False, 0), (True, 0), (True, 1)])
+def test_inkling2_responses_function_calls(client, stream, chunk):
+ server.response_store.clear()
+ server.response_store_order.clear()
+ response, _ = _inkling2_request(
+ client,
+ INKLING2_THINKING + INKLING2_TEXT + _INKLING2_CALLS,
+ "responses",
+ stream=stream,
+ chunk=chunk,
+ tools=[dict(type="function", name="get_weather", parameters={})],
+ )
+ body = _completed_response(response) if stream else response.json()
+ items = {}
+ for item in body["output"]:
+ items.setdefault(item["type"], []).append(item)
+ calls = items.pop("function_call")
+ assert [
+ (call["name"], json.loads(call["arguments"])) for call in calls
+ ] == _INKLING2_WEATHER_CALLS
+ assert len({call["call_id"] for call in calls}) == 2
+ (reasoning,) = items.pop("reasoning")
+ assert reasoning["summary"][0]["text"] == "I need both cities."
+ # The text beside the calls is a message item only when streamed.
+ for message in items.pop("message", []):
+ assert message["content"][0]["text"] == "Lisbon is warmer."
+ assert not items
+ assert body["output_text"] == "Lisbon is warmer."
+ assert body["status"] == "completed"
+ assert "<|" not in response.text
+
+
+@pytest.mark.parametrize("api", ["messages", "responses"])
+@pytest.mark.parametrize("stream", [False, True])
+def test_inkling2_truncated_call_ends_the_turn_cleanly(client, api, stream):
+ server.response_store.clear()
+ server.response_store_order.clear()
+ tool = (
+ _tool(api="messages")
+ if api == "messages"
+ else dict(type="function", name="get_weather", parameters={})
+ )
+ response, _ = _inkling2_request(
+ client,
+ INKLING2_TEXT + _inkling2_call("get_weather", city="Lisbon")[:-24],
+ api,
+ stream=stream,
+ chunk=4 * stream,
+ finish_reason="length",
+ tools=[tool],
+ )
+ if api == "responses":
+ body = _completed_response(response) if stream else response.json()
+ assert body["status"] == "incomplete"
+ assert [item["type"] for item in body["output"]] == ["message"]
+ assert body["output_text"] == "Lisbon is warmer."
+ elif stream:
+ assert _thinking_text(response, api) == ("", "Lisbon is warmer.")
+ assert _anthropic_stream_stop(response) == ("max_tokens", None)
+ else:
+ body = response.json()
+ assert body["content"] == [dict(type="text", text="Lisbon is warmer.")]
+ assert body["stop_reason"] == "max_tokens"
+ assert "<|" not in response.text and "city" not in response.text
+
+
+_INKLING2_HISTORY = dict(
+ chat=dict(
+ messages=[
+ _msg("Compare Lisbon and Oslo."),
+ _msg(
+ None,
+ "assistant",
+ reasoning_content="I need both cities.",
+ tool_calls=[
+ dict(
+ id=f"call_{city}",
+ type="function",
+ # Clients send the arguments back as a JSON string.
+ function=dict(
+ name="get_weather", arguments=json.dumps(dict(city=city))
+ ),
+ )
+ for city in ("Lisbon", "Oslo")
+ ],
+ ),
+ _msg("19C", "tool", tool_call_id="call_Lisbon"),
+ _msg("3C", "tool", tool_call_id="call_Oslo"),
+ ],
+ tools=[_tool()],
+ ),
+ responses=dict(
+ input=[
+ _input_message("Compare Lisbon and Oslo."),
+ *(
+ dict(
+ type="function_call",
+ name="get_weather",
+ arguments=json.dumps(dict(city=city)),
+ call_id=f"call_{city}",
+ )
+ for city in ("Lisbon", "Oslo")
+ ),
+ dict(type="function_call_output", call_id="call_Lisbon", output="19C"),
+ dict(type="function_call_output", call_id="call_Oslo", output="3C"),
+ ],
+ tools=[dict(type="function", name="get_weather", parameters={})],
+ ),
+ messages=dict(
+ messages=[
+ _msg("Compare Lisbon and Oslo."),
+ _msg(
+ [
+ dict(
+ type="tool_use",
+ id=f"call_{city}",
+ name="get_weather",
+ input=dict(city=city),
+ )
+ for city in ("Lisbon", "Oslo")
+ ],
+ "assistant",
+ ),
+ _msg(
+ [
+ dict(type="tool_result", tool_use_id="call_Lisbon", content="19C"),
+ dict(type="tool_result", tool_use_id="call_Oslo", content="3C"),
+ ]
+ ),
+ ],
+ tools=[_tool(api="messages")],
+ max_tokens=64,
+ ),
+)
+
+
+@pytest.mark.parametrize("api", ["chat", "responses", "messages"])
+def test_inkling2_history_with_string_arguments_renders(client, api):
+ # The chat template refuses string arguments, so the stored history must
+ # reach it as objects, and each tool result must find its call's name.
+ _, prompts = _inkling2_request(client, INKLING2_TEXT, api, **_INKLING2_HISTORY[api])
+ prompt = prompts[-1]
+ assert prompt.startswith("<|message_system|>tool_declare<|content_xml|>[{")
+ rendered = [
+ _inkling2_call("get_weather", city="Lisbon"),
+ _inkling2_call("get_weather", city="Oslo"),
+ "<|message_tool|>get_weather<|content_text|>19C<|end_message|>",
+ "<|message_tool|>get_weather<|content_text|>3C<|end_message|>",
+ ]
+ assert all(prompt.count(part) == 1 for part in rendered)
+ assert sorted(rendered, key=prompt.index) == rendered
+ if api == "chat":
+ assert "<|content_thinking|>I need both cities.<|end_message|>" in prompt
+
+
+@pytest.mark.parametrize("api", ["chat", "responses"])
+@pytest.mark.parametrize(
+ "effort,level",
+ [
+ (None, "0.9"),
+ ("none", "0"),
+ ("minimal", "0.1"),
+ ("low", "0.2"),
+ ("medium", "0.7"),
+ ("high", "0.9"),
+ ("xhigh", "0.99"),
+ ("max", "0.99"),
+ ],
+)
+def test_inkling2_reasoning_effort_reaches_template(client, api, effort, level):
+ payload = {}
+ if effort is not None:
+ payload = (
+ dict(reasoning_effort=effort)
+ if api == "chat"
+ else dict(reasoning=dict(effort=effort))
+ )
+ _, prompts = _inkling2_request(client, INKLING2_TEXT, api, **payload)
+ assert f"Thinking effort level: {level}<|end_message|>" in prompts[-1]
+
+
+@pytest.mark.parametrize("stream", [False, True])
+def test_inkling2_split_survives_an_undecodable_call(stream):
+ # The response template decodes a call's JSON when its region closes, and
+ # raises on a truncated or malformed one. That must not cost the turn its
+ # reasoning and content, nor put the raw text into the content.
+ processor = NS(
+ tokenizer=fast_tokenizer(response_template=_INKLING2_RESPONSE_TEMPLATE)
+ )
+ for call in (
+ _inkling2_call("get_weather", city="Lisbon")[:-24],
+ _inkling2_call("get_weather", raw="{city}"),
+ _inkling2_call("get_weather", raw='{"name":"get_weather"}'),
+ ):
+ text = INKLING2_THINKING + call + INKLING2_TEXT
+ closed = call.endswith("<|end_message|>")
+ expected = ("I need both cities.", "Lisbon is warmer." if closed else "")
+ if stream:
+ state = server.make_response_stream_state(processor)
+ assert _thoughts(_feed_thinking(state, list(text), last=True)) == expected
+ else:
+ assert server._split_thinking(text, processor=processor) == expected
+
+
def test_chat_completions_endpoint_falls_back_from_video_to_images(client):
from mlx_vlm.generate import video
diff --git a/mlx_vlm/tools/base.py b/mlx_vlm/tools/base.py
index de8f5486..7432dcb5 100644
--- a/mlx_vlm/tools/base.py
+++ b/mlx_vlm/tools/base.py
@@ -6,6 +6,12 @@ string for formats whose call ends at a newline rather than a closing marker.
The extractor (:mod:`mlx_vlm.tools.extractor`) drives any module shaped like
this; the :class:`ParserSpec` in :mod:`mlx_vlm.tools.registry` maps a chat
template to the module that owns it.
+
+A format whose calls two markers cannot delimit may also define
+``find_tool_calls(text) -> [(start, end), ...]``. It receives the whole
+assistant turn, reasoning included, and returns the span of every executable
+call in order; the extractor then uses those spans instead of the markers
+(see :mod:`mlx_vlm.tools.parsers.inkling2`).
"""
from __future__ import annotations
diff --git a/mlx_vlm/tools/extractor.py b/mlx_vlm/tools/extractor.py
index 2ff57b66..36b3bdd2 100644
--- a/mlx_vlm/tools/extractor.py
+++ b/mlx_vlm/tools/extractor.py
@@ -1,15 +1,16 @@
"""Extract structured tool calls from a model's text output.
Format-agnostic: given the parser module selected for the model (see
-:mod:`mlx_vlm.tools.registry`), it slices the text between the parser's markers,
-delegates per-format decoding to ``parse_tool_call``, and emits OpenAI-shaped
-calls.
+:mod:`mlx_vlm.tools.registry`), it slices the text between the parser's markers
+(or at the spans the parser's own ``find_tool_calls`` reports), delegates
+per-format decoding to ``parse_tool_call``, and emits OpenAI-shaped calls.
"""
import json
import logging
import re
import uuid
+from typing import List, Tuple
from .base import ToolParser
from .types import ParseResult
@@ -17,6 +18,23 @@ from .types import ParseResult
logger = logging.getLogger("mlx_vlm.server")
+def _call_spans(model_output: str, tool_module: ToolParser) -> List[Tuple[int, int]]:
+ """Return the ``(start, end)`` span of every tool call, markers included."""
+ find_tool_calls = getattr(tool_module, "find_tool_calls", None)
+ if find_tool_calls is not None:
+ return find_tool_calls(model_output)
+ if tool_module.tool_call_end == "":
+ pattern = re.compile(
+ f"{re.escape(tool_module.tool_call_start)}.*?(?:\n|$)", re.DOTALL
+ )
+ else:
+ pattern = re.compile(
+ f"{re.escape(tool_module.tool_call_start)}.*?{re.escape(tool_module.tool_call_end)}",
+ re.DOTALL,
+ )
+ return [match.span() for match in pattern.finditer(model_output)]
+
+
def process_tool_calls(
model_output: str, tool_module: ToolParser, tools
) -> ParseResult:
@@ -25,19 +43,14 @@ def process_tool_calls(
remaining = model_output
if tool_module.tool_call_start in model_output:
- if tool_module.tool_call_end == "":
- pattern = re.compile(
- f"{re.escape(tool_module.tool_call_start)}.*?(?:\n|$)", re.DOTALL
- )
- else:
- pattern = re.compile(
- f"{re.escape(tool_module.tool_call_start)}.*?{re.escape(tool_module.tool_call_end)}",
- re.DOTALL,
- )
-
- matches = re.findall(pattern, model_output)
- if matches:
- remaining = re.sub(pattern, " ", model_output).strip()
+ spans = _call_spans(model_output, tool_module)
+ if spans:
+ matches = [model_output[start:end] for start, end in spans]
+ # Each call is replaced with a space.
+ edges = [0, *(edge for span in spans for edge in span), len(model_output)]
+ remaining = " ".join(
+ model_output[start:end] for start, end in zip(edges[::2], edges[1::2])
+ ).strip()
for match in matches:
call = (
match.strip()
diff --git a/mlx_vlm/tools/parsers/inkling2.py b/mlx_vlm/tools/parsers/inkling2.py
new file mode 100644
index 00000000..1f9d18f0
--- /dev/null
+++ b/mlx_vlm/tools/parsers/inkling2.py
@@ -0,0 +1,99 @@
+"""Inkling 2 tool calls.
+
+Inkling 2 writes an assistant turn as a run of messages, each opened by a role
+header and a content-type token and closed by ``<|end_message|>``::
+
+ <|message_model|><|content_thinking|>I need the weather.<|end_message|>
+ <|message_model|><|content_text|>Let me check.<|end_message|>
+ <|message_model|>get_weather<|content_invoke_tool_json|>{"name":"get_weather","args":{"city":"Oslo"}}<|end_message|>
+
+A tool call is a message of type ``invoke_tool_json``: its header repeats the
+tool name and its body is one JSON object with ``name`` and ``args``. Parallel
+calls are consecutive call messages.
+
+The two markers alone do not delimit a call: the header belongs to it, and the
+closing token also closes every other message. :func:`find_tool_calls` therefore
+walks the turn message by message, the way the tokenizer's ``response_template``
+does, so call markup quoted inside a reasoning or text message is not a call.
+
+A call whose body is not a JSON object with a string ``name`` and an object
+``args`` is rejected: the extractor logs and drops it, and keeps the calls
+around it. A call still open when generation stops (``max_tokens``) is not a
+call.
+
+Limit: this works on decoded text, where a control token and the same characters
+written as ordinary text look alike. A call body is JSON, so ``<|end_message|>``
+inside one of its strings is recognised as data (see :func:`_call_end`). In a
+reasoning or text message those characters end the message. The server splits
+reasoning and content with the response template, which does not read JSON: a
+message opening such as ``<|content_text|>`` written inside an argument string
+after those characters also shows up as content.
+"""
+
+import json
+import re
+from typing import Any, List, Optional, Tuple
+
+tool_call_start = "<|content_invoke_tool_json|>"
+tool_call_end = "<|end_message|>"
+
+_HEADER = r"(?:<\|message_model\|>[^<]*)?"
+# The message openings the response template knows; other text is not a message.
+_MESSAGE = re.compile(
+ _HEADER
+ + r"<\|content_(thinking|text|tool_error|invoke_tool_text|invoke_tool_json)\|>"
+)
+_CALL_OPENING = re.compile(rf"\s*{_HEADER}{re.escape(tool_call_start)}")
+
+
+def _call_end(text: str, start: int) -> int:
+ """Index of the ``<|end_message|>`` that closes the call body at ``start``.
+
+ The call ends at the first marker that leaves valid JSON before it, so the
+ marker's characters inside a JSON string stay part of the arguments. A body
+ that is valid at no marker ends at the first one, where
+ :func:`parse_tool_call` rejects it. Returns -1 while the call is unclosed.
+ """
+ first = end = text.find(tool_call_end, start)
+ while end >= 0:
+ try:
+ json.loads(text[start:end])
+ return end
+ except ValueError:
+ end = text.find(tool_call_end, end + len(tool_call_end))
+ return first
+
+
+def find_tool_calls(text: str) -> List[Tuple[int, int]]:
+ """Return the ``(start, end)`` span of every call message, header included.
+
+ A message still open when the text ends (generation was cut off) is not a
+ call, and nothing after it is.
+ """
+ spans = []
+ position = 0
+ while (message := _MESSAGE.search(text, position)) is not None:
+ is_call = message[1] == "invoke_tool_json"
+ end = (
+ _call_end(text, message.end())
+ if is_call
+ else text.find(tool_call_end, message.end())
+ )
+ if end < 0:
+ break
+ position = end + len(tool_call_end)
+ if is_call:
+ spans.append((message.start(), position))
+ return spans
+
+
+def parse_tool_call(text: str, tools: Optional[Any] = None) -> dict:
+ """Decode one call body, with or without its ``<|message_model|>NAME`` opening."""
+ opening = _CALL_OPENING.match(text)
+ call = json.loads(text[opening.end() :] if opening else text)
+ if not isinstance(call, dict) or not isinstance(call.get("name"), str):
+ raise ValueError("Inkling 2 tool call must be an object with a name.")
+ arguments = call.get("args", {})
+ if not isinstance(arguments, dict):
+ raise ValueError("Inkling 2 tool call args must be an object.")
+ return {"name": call["name"], "arguments": arguments}
diff --git a/mlx_vlm/tools/registry.py b/mlx_vlm/tools/registry.py
index ddf3b278..8f4447f7 100644
--- a/mlx_vlm/tools/registry.py
+++ b/mlx_vlm/tools/registry.py
@@ -65,6 +65,7 @@ SPECS: Tuple[ParserSpec, ...] = (
ParserSpec("kimi_k2", (("<|tool_calls_section_begin|>",),)),
ParserSpec("mistral", (("[TOOL_CALLS]",),)),
ParserSpec("harmony", (("<|channel|>commentary", "<|call|>"),)),
+ ParserSpec("inkling2", (("<|content_invoke_tool_json|>",),)),
ParserSpec("json_tools", (("<tool_call>", "tool_call.name"),), priority=0),
)
diff --git a/mlx_vlm/server/anthropic.py b/mlx_vlm/server/anthropic.py
index ed46ff05..1b714bec 100644
--- a/mlx_vlm/server/anthropic.py
+++ b/mlx_vlm/server/anthropic.py
from .generation import (
from .responses_state import (
ThinkingStreamState,
ToolCallStreamState,
+ content_beside_tool_calls,
finish_content_streams,
make_response_stream_state,
parse_assistant_tool_calls,
prompt_has_open_thinking,
- strip_protocol_markers,
)
from .runtime import runtime
from .schemas import AnthropicMessageResponse, AnthropicRequest, AnthropicUsage
async def anthropic_messages_endpoint(http_request: Request):
tc = parse_tools(full_text)
if tc.calls:
parsed_tool_calls = tc.calls
- # Structured templates already removed tool/routing regions
- # from the content parsed above. Otherwise use the delimiter
- # parser's remainder (reasoning has already been excluded).
- if not (
- getattr(processor, "response_template", None)
- or getattr(
- getattr(processor, "tokenizer", None),
- "response_template",
- None,
- )
- ):
- content = tc.remaining_text or ""
- content = strip_protocol_markers(
+ content = content_beside_tool_calls(
content,
+ tc.remaining_text,
+ processor,
tool_module,
gen_args.thinking_start_token,
gen_args.thinking_end_token,
diff --git a/mlx_vlm/server/openai.py b/mlx_vlm/server/openai.py
index a1729e13..3d4d0048 100644
--- a/mlx_vlm/server/openai.py
+++ b/mlx_vlm/server/openai.py
from .responses_state import (
from .responses_state import _sse_event as _response_sse_event
from .responses_state import (
_store_response,
+ content_beside_tool_calls,
finish_content_streams,
make_response_stream_state,
parse_assistant_tool_calls,
prompt_has_open_thinking,
response_store,
response_store_lock,
- strip_protocol_markers,
)
from .runtime import runtime
from .schemas import (
async def chat_completions_endpoint(request: ChatRequest, http_request: Request)
tc = parse_tools(full_text)
if tc.calls:
parsed_tool_calls = tc.calls
- # Reasoning was already removed before tool parsing.
- clean_remaining = tc.remaining_text or ""
content = (
- strip_protocol_markers(
- clean_remaining,
+ content_beside_tool_calls(
+ content,
+ tc.remaining_text,
+ processor,
tool_module,
gen_args.thinking_start_token,
gen_args.thinking_end_token,
diff --git a/mlx_vlm/server/responses_state.py b/mlx_vlm/server/responses_state.py
index d975a822..424a54ea 100644
--- a/mlx_vlm/server/responses_state.py
+++ b/mlx_vlm/server/responses_state.py
from threading import Lock
from typing import Any, Dict, List, Optional, Tuple
from fastapi import HTTPException
+from transformers.utils.chat_parsing import ResponseParser, parse_response
from transformers.utils.chat_parsing.response_templates import load_response_template
from ..prompt_utils import _normalize_tool_message
def _response_template_prefix(tokenizer, prompt: str) -> str:
return prompt if template.start_anchor_re.search(prompt) else ""
+_TEXT_FIELDS = ("content", "reasoning", "reasoning_content")
+_REGION_KEYS = ("open", "open_pattern", "close", "close_pattern", "repeats")
+
+
+def _text_response_template(template: Dict[str, Any]) -> Dict[str, Any]:
+ """Copy a response template, reading its structured fields as plain text.
+
+ The server takes reasoning and content from a response template; tool calls