Inkling-2-Mini-MLX-4bit / extras /mlx-vlm-inkling2-tool-calls.patch
VisualInference's picture
Inkling 2 Mini for MLX, 4bit: weights, card, validation results
d19188c verified
Raw History Blame Contribute Delete
49.4 kB
diff --git a/mlx_vlm/server/anthropic.py b/mlx_vlm/server/anthropic.py index ed46ff05..1b714bec 100644 --- a/mlx_vlm/server/anthropic.py +++ b/mlx_vlm/server/anthropic.py @@ -29,11 +29,11 @@ from .generation import ( from .responses_state import ( ThinkingStreamState, ToolCallStreamState, + content_beside_tool_calls, finish_content_streams, make_response_stream_state, parse_assistant_tool_calls, prompt_has_open_thinking, - strip_protocol_markers, ) from .runtime import runtime from .schemas import AnthropicMessageResponse, AnthropicRequest, AnthropicUsage @@ -1130,20 +1130,10 @@ async def anthropic_messages_endpoint(http_request: Request): tc = parse_tools(full_text) if tc.calls: parsed_tool_calls = tc.calls - # Structured templates already removed tool/routing regions - # from the content parsed above. Otherwise use the delimiter - # parser's remainder (reasoning has already been excluded). - if not ( - getattr(processor, "response_template", None) - or getattr( - getattr(processor, "tokenizer", None), - "response_template", - None, - ) - ): - content = tc.remaining_text or "" - content = strip_protocol_markers( + content = content_beside_tool_calls( content, + tc.remaining_text, + processor, tool_module, gen_args.thinking_start_token, gen_args.thinking_end_token, diff --git a/mlx_vlm/server/openai.py b/mlx_vlm/server/openai.py index a1729e13..3d4d0048 100644 --- a/mlx_vlm/server/openai.py +++ b/mlx_vlm/server/openai.py @@ -60,13 +60,13 @@ from .responses_state import ( from .responses_state import _sse_event as _response_sse_event from .responses_state import ( _store_response, + content_beside_tool_calls, finish_content_streams, make_response_stream_state, parse_assistant_tool_calls, prompt_has_open_thinking, response_store, response_store_lock, - strip_protocol_markers, ) from .runtime import runtime from .schemas import ( @@ -2807,11 +2807,11 @@ async def chat_completions_endpoint(request: ChatRequest, http_request: Request) tc = parse_tools(full_text) if tc.calls: parsed_tool_calls = tc.calls - # Reasoning was already removed before tool parsing. - clean_remaining = tc.remaining_text or "" content = ( - strip_protocol_markers( - clean_remaining, + content_beside_tool_calls( + content, + tc.remaining_text, + processor, tool_module, gen_args.thinking_start_token, gen_args.thinking_end_token, diff --git a/mlx_vlm/server/responses_state.py b/mlx_vlm/server/responses_state.py index d975a822..424a54ea 100644 --- a/mlx_vlm/server/responses_state.py +++ b/mlx_vlm/server/responses_state.py @@ -9,6 +9,7 @@ from threading import Lock from typing import Any, Dict, List, Optional, Tuple from fastapi import HTTPException +from transformers.utils.chat_parsing import ResponseParser, parse_response from transformers.utils.chat_parsing.response_templates import load_response_template from ..prompt_utils import _normalize_tool_message @@ -241,6 +242,29 @@ def _response_template_prefix(tokenizer, prompt: str) -> str: return prompt if template.start_anchor_re.search(prompt) else "" +_TEXT_FIELDS = ("content", "reasoning", "reasoning_content") +_REGION_KEYS = ("open", "open_pattern", "close", "close_pattern", "repeats") + + +def _text_response_template(template: Dict[str, Any]) -> Dict[str, Any]: + """Copy a response template, reading its structured fields as plain text. + + The server takes reasoning and content from a response template; tool calls + come from the tool parser. A structured field such as a JSON tool call is + decoded when its region closes, and a malformed or truncated one raises + there, which would lose the reasoning and content around it. Those fields + keep their delimiters, so their text still stays out of the content. + """ + fields = {} + for name, field in template["fields"].items(): + if name not in _TEXT_FIELDS: + field = {key: field[key] for key in _REGION_KEYS if key in field} + # A transform is what lets an open pattern keep its named groups. + field["transform"] = "{content}" + fields[name] = field + return {**template, "fields": fields} + + def make_response_stream_state( processor, enable_thinking: bool = False, @@ -253,8 +277,9 @@ def make_response_stream_state( if tokenizer is not None and hasattr(tokenizer, "get_response_parser"): try: return ResponseTemplateStreamState( - tokenizer.get_response_parser( - prefix=_response_template_prefix(tokenizer, prompt) + ResponseParser( + _text_response_template(tokenizer.response_template), + prefix=_response_template_prefix(tokenizer, prompt), ) ) except (AttributeError, TypeError, ValueError): @@ -438,6 +463,28 @@ def strip_protocol_markers( return text.strip() +def content_beside_tool_calls( + content: Optional[str], + remaining_text: Optional[str], + processor: Any, + tool_module: Any, + thinking_start_token: Optional[str] = None, + thinking_end_token: Optional[str] = None, +) -> str: + """Return the visible content of a turn that also made tool calls. + + ``content`` is what ``_split_thinking`` returned for the turn. A response + template already kept tool calls and message routing out of it. Without a + template, the content is ``remaining_text``, the tool parser's remainder + (reasoning was excluded before tool parsing). + """ + if _response_template_tokenizer(processor) is None: + content = remaining_text + return strip_protocol_markers( + content or "", tool_module, thinking_start_token, thinking_end_token + ) + + def finish_content_streams(thinking_state, tool_call_state): """``(reasoning, content)`` still held when a stream ends. @@ -494,13 +541,23 @@ def _split_thinking( tokenizer = _response_template_tokenizer(processor) if tokenizer is not None and hasattr(tokenizer, "parse_response"): try: - parsed = tokenizer.parse_response( - text, prefix=_response_template_prefix(tokenizer, prompt) - ) + prefix = _response_template_prefix(tokenizer, prompt) + try: + parsed = tokenizer.parse_response(text, prefix=prefix) + except ValueError: + # A structured field, such as a truncated tool call, did not + # decode. The reasoning and content around it are still valid. + parsed = parse_response( + text, + _text_response_template(tokenizer.response_template), + prefix=prefix, + ) + # A turn of tool calls alone is a parsed turn with no content. if isinstance(parsed, dict) and ( "content" in parsed or "reasoning" in parsed or "reasoning_content" in parsed + or "tool_calls" in parsed ): reasoning = parsed.get("reasoning_content") or parsed.get("reasoning") content = parsed.get("content") @@ -556,7 +613,10 @@ def parse_assistant_tool_calls( Response-template parsers may consume tool regions, so split reasoning with the delimiter state instead. Examples inside reasoning are never tool calls. + A parser with ``find_tool_calls`` locates calls in the whole turn itself. """ + if hasattr(tool_module, "find_tool_calls"): + return process_tool_calls(text or "", tool_module, tools) content = ( ThinkingStreamState( starts_in_thinking, thinking_start_token, thinking_end_token @@ -606,8 +666,10 @@ def _response_output_items_from_text( items = [ _tool_call_to_response_item(call, tool_registry) for call in tc.calls ] - remaining = strip_protocol_markers( - tc.remaining_text or "", + remaining = content_beside_tool_calls( + content, + tc.remaining_text, + processor, tool_module, thinking_start_token, thinking_end_token, diff --git a/mlx_vlm/tests/test_processors.py b/mlx_vlm/tests/test_processors.py index e325391c..8f4cb6ad 100644 --- a/mlx_vlm/tests/test_processors.py +++ b/mlx_vlm/tests/test_processors.py @@ -2972,6 +2972,8 @@ WIRE_CALLS = { "mistral": '[TOOL_CALLS]get_weather[ARGS]{"city": "Paris", "days": 3}', "harmony": "<|channel|>commentary to=functions.get_weather <|constrain|>json" '<|message|>{"city": "Paris", "days": 3}<|call|>', + "inkling2": "<|message_model|>get_weather<|content_invoke_tool_json|>" + '{"name":"get_weather","args":{"city":"Paris","days":3}}<|end_message|>', "pythonic": '<|tool_call_start|>[get_weather(city="Paris", days=3)]<|tool_call_end|>', "qwen3_coder": "<tool_call>\n<function=get_weather><parameter=city>Paris</parameter>" "<parameter=days>3</parameter></function></tool_call>", @@ -3162,6 +3164,9 @@ def test_deepseek_streamed_tool_calls(chunk_size): ("minicpm5", '<function name="lookup"><param>3</param></function>', None), ("gemma4", "just a normal model response, no tool call here", None), ("mistral", "not a tool call at all", None), + ("inkling2", '{"name":"get_weather","args":{"city":"Par', None), + ("inkling2", '{"args":{"city":"Paris"}}', "object with a name"), + ("inkling2", '{"name":"get_weather","args":["Paris"]}', "must be an object"), ( "pythonic", "[write_file(content='const player = { x: 0, y: 1 };)]", @@ -3527,6 +3532,130 @@ def test_harmony_ignores_plain_commentary_preamble(): assert result.remaining_text == preamble +def _inkling2_call(name, raw=None, **arguments): + body = raw or json.dumps( + dict(name=name, args=arguments), ensure_ascii=False, separators=(",", ":") + ) + return f"<|message_model|>{name}<|content_invoke_tool_json|>{body}<|end_message|>" + + +def _inkling2_calls(text): + result = process_tool_calls(text, load_tool_module("inkling2"), None) + calls = [ + (call["function"]["name"], json.loads(call["function"]["arguments"])) + for call in result.calls + ] + return calls, result + + +INKLING2_THINKING = ( + "<|message_model|><|content_thinking|>I need both cities.<|end_message|>" +) +INKLING2_TEXT = "<|message_model|><|content_text|>Lisbon is warmer.<|end_message|>" +INKLING2_ARGUMENTS = { + "nested": {"list": [1, 2.5, None, True, {"deep": []}], "empty": {}}, + "unicode": "19°C — ação 日本 🐦", + "quotes": 'say "hi" \\ back\\slash', + "lines": "first\nsecond\ttabbed\r\n", + "markers": "<|end_message|><|message_model|>x<|content_invoke_tool_json|>{}", + "tags": "<think> </tool_call>", +} + + +def test_inkling2_parallel_calls_follow_reasoning(): + text = ( + INKLING2_THINKING + + _inkling2_call("get_weather", city="Lisbon", unit="celsius") + + _inkling2_call("get_weather", city="Oslo", unit="celsius") + ) + calls, result = _inkling2_calls(text) + assert calls == [ + ("get_weather", {"city": "Lisbon", "unit": "celsius"}), + ("get_weather", {"city": "Oslo", "unit": "celsius"}), + ] + assert [call["index"] for call in result.calls] == [0, 1] + assert len({call["id"] for call in result.calls}) == 2 + # The role header and the tool name it carries leave with the call. + assert result.remaining_text == INKLING2_THINKING + + +@pytest.mark.parametrize( + "text,expected", + [ + (INKLING2_THINKING + INKLING2_TEXT, []), + (INKLING2_TEXT + _inkling2_call("get_time"), [("get_time", {})]), + ( + "<|content_invoke_tool_json|>" + '{"name":"get_time"}' + "<|end_message|>", + [("get_time", {})], + ), + ( + # Call markup quoted inside reasoning or text is not a call. + "<|message_model|><|content_thinking|>Not " + + _inkling2_call("delete_all") + + INKLING2_TEXT, + [], + ), + ( + "<|message_model|><|content_text|>Example: <|content_invoke_tool_json|>" + '{"name":"delete_all","args":{}}<|end_message|>' + + _inkling2_call("get_time"), + [("get_time", {})], + ), + ], + ids=["text-only", "text-then-call", "bare-call", "in-reasoning", "in-text"], +) +def test_inkling2_calls_are_whole_messages(text, expected): + assert _inkling2_calls(text)[0] == expected + + +@pytest.mark.parametrize("ascii_only", [False, True]) +def test_inkling2_arguments_round_trip(ascii_only): + body = json.dumps( + dict(name="write_file", args=INKLING2_ARGUMENTS), ensure_ascii=ascii_only + ) + text = ( + _inkling2_call("write_file", raw=body) + + INKLING2_TEXT + + _inkling2_call("get_time") + ) + calls, result = _inkling2_calls(text) + assert calls == [("write_file", INKLING2_ARGUMENTS), ("get_time", {})] + assert result.remaining_text == INKLING2_TEXT + # Non-ASCII arguments are returned as written, not as escapes. + assert "19°C — ação 日本 🐦" in result.calls[0]["function"]["arguments"] + + +@pytest.mark.parametrize( + "broken", + [ + '{"name":"get_weather","args":{city}}', + '{"name":"get_weather","args":{"city":"unterminated}}', + '{"name":"get_weather","args":"Paris"}', + '["get_weather"]', + "", + ], +) +def test_inkling2_malformed_call_is_dropped_alone(broken): + text = ( + _inkling2_call("get_time") + + _inkling2_call("get_weather", raw=broken or " ") + + INKLING2_TEXT + + _inkling2_call("get_date") + ) + calls, result = _inkling2_calls(text) + assert calls == [("get_time", {}), ("get_date", {})] + assert result.remaining_text == INKLING2_TEXT + + +@pytest.mark.parametrize("cut", range(1, 60, 3)) +def test_inkling2_truncated_call_is_not_a_call(cut): + complete = INKLING2_THINKING + _inkling2_call("get_time") + call = _inkling2_call("get_weather", city="Lisbon") + calls, result = _inkling2_calls(complete + call[:-cut]) + assert calls == [("get_time", {})] + assert "get_time" not in result.remaining_text + + # Loading and utility contracts diff --git a/mlx_vlm/tests/test_server.py b/mlx_vlm/tests/test_server.py index 59eb1e94..6e44662b 100644 --- a/mlx_vlm/tests/test_server.py +++ b/mlx_vlm/tests/test_server.py @@ -73,7 +73,14 @@ from mlx_vlm.server.responses_state import ( strip_protocol_markers, ) from mlx_vlm.server.runtime_config import RuntimeConfig -from mlx_vlm.tests.test_processors import MINICPM_MULTICALL +from mlx_vlm.tests.test_processors import ( + INKLING2_ARGUMENTS, + INKLING2_TEXT, + INKLING2_THINKING, + MINICPM_MULTICALL, + _inkling2_call, + fast_tokenizer, +) from mlx_vlm.tokenizer_utils import SPMStreamingDetokenizer, _ServerTokenStreamer from mlx_vlm.tools import load_tool_module, process_tool_calls @@ -3680,6 +3687,631 @@ def test_response_template_tool_calls(client, api): assert "to=self" not in response.text and "<atem:" not in response.text +@pytest.mark.parametrize("api", ["chat", "responses"]) +def test_response_template_content_excludes_tool_routing(client, api): + # Non-streamed, like the stream: a response template already kept the + # reasoning and the call's routing header out of the content. + config = NS(model_type="muse_glimmer") + tool = ( + dict(type="function", name="get_weather", parameters={"type": "object"}) + if api == "responses" + else _tool() + ) + with _endpoint( + config=config, + processor=NS(config=config, tokenizer=_MuseResponseTemplateTokenizer()), + result=_result(_MUSE_CALL), + parser=load_tool_module("atem"), + ): + response = _post(client, api, tools=[tool]) + body = response.json() + if api == "chat": + message = body["choices"][0]["message"] + assert message["content"] is None + assert message["reasoning_content"] == "I need the weather tool." + assert message["tool_calls"][0]["function"]["name"] == "get_weather" + else: + assert body["output_text"] == "" + assert [item["type"] for item in body["output"]] == [ + "reasoning", + "function_call", + ] + assert "to=" not in response.text + + +# Inkling 2 tool calls. The tokenizer is a real one carrying the response +# template the model ships and a chat template condensed from the model's: the +# same message framing, and the same refusal of string tool-call arguments. + +_INKLING2_OPEN = r"(?:<\|message_model\|>[^<]*)?<\|content_%s\|>" +_INKLING2_RESPONSE_TEMPLATE = { + "defaults": {"role": "assistant"}, + "start_anchor_pattern": r"<\|end_message\|>|<\|content_model_end_sampling\|>", + "fields": { + "reasoning_content": { + "open_pattern": _INKLING2_OPEN % "thinking", + "close": "<|end_message|>", + "repeats": True, + "join": "", + "content": "text", + "content_args": {"strip": False}, + }, + "content": { + "open_pattern": _INKLING2_OPEN % "(?:text|tool_error|invoke_tool_text)", + "close": "<|end_message|>", + "repeats": True, + "join": "", + "content": "text", + "content_args": {"strip": False}, + }, + "tool_calls": { + "open_pattern": _INKLING2_OPEN % "invoke_tool_json", + "close": "<|end_message|>", + "repeats": True, + "content": "json", + "transform": { + "type": "function", + "function": {"name": "{content.name}", "arguments": "{content.args}"}, + }, + }, + }, +} +_INKLING2_CHAT_TEMPLATE = """ +{%- set levels = {"none": "0", "minimal": "0.1", "low": "0.2", "medium": "0.7", + "high": "0.9", "xhigh": "0.99", "max": "0.99"} -%} +{%- set roles = {"user": "<|message_user|>", "assistant": "<|message_model|>", + "system": "<|message_system|>", "tool": "<|message_tool|>"} -%} +{%- if tools -%} + {{- "<|message_system|>tool_declare<|content_xml|>" ~ (tools | tojson) -}} + {{- "<|end_message|>" -}} +{%- endif -%} +{%- set effort = reasoning_effort if reasoning_effort is defined + and reasoning_effort is not none else "high" -%} +{%- if effort not in levels -%} + {{- raise_exception("Unknown reasoning_effort: " ~ effort) -}} +{%- endif -%} +{{- "<|message_system|><|content_text|>Thinking effort level: " ~ levels[effort] -}} +{{- "<|end_message|>" -}} +{%- for message in messages -%} + {%- set header = namespace(text=roles[message.role]) -%} + {%- if message.role == "tool" -%} + {%- for turn in messages if turn.role == "assistant" and turn.tool_calls -%} + {%- for call in turn.tool_calls if call.id == message.tool_call_id -%} + {%- set header.text = header.text ~ call.function.name -%} + {%- endfor -%} + {%- endfor -%} + {%- endif -%} + {%- if message.role == "assistant" and message.reasoning_content -%} + {{- header.text ~ "<|content_thinking|>" ~ message.reasoning_content -}} + {{- "<|end_message|>" -}} + {%- endif -%} + {%- if message.content is string -%} + {%- if message.content -%} + {{- header.text ~ "<|content_text|>" ~ message.content -}} + {{- "<|end_message|>" -}} + {%- endif -%} + {%- else -%} + {%- for part in message.content or [] -%} + {{- header.text ~ "<|content_text|>" ~ part.text ~ "<|end_message|>" -}} + {%- endfor -%} + {%- endif -%} + {%- for call in message.tool_calls or [] -%} + {%- if call.function.arguments is string -%} + {{- raise_exception("tool call arguments must be a parsed object") -}} + {%- endif -%} + {{- "<|message_model|>" ~ call.function.name -}} + {{- '<|content_invoke_tool_json|>{"name":' ~ (call.function.name | tojson) -}} + {{- ',"args":' ~ (call.function.arguments | tojson(separators=(",", ":"))) -}} + {{- "}<|end_message|>" -}} + {%- endfor -%} + {%- if message.role == "assistant" -%} + {{- "<|content_model_end_sampling|>" -}} + {%- endif -%} +{%- endfor -%} +""" +_INKLING2_LISBON = dict(city="Lisbon", unit="celsius") +_INKLING2_OSLO = dict(city="Oslo", unit="celsius") +_INKLING2_CALLS = _inkling2_call("get_weather", **_INKLING2_LISBON) + _inkling2_call( + "get_weather", **_INKLING2_OSLO +) +_INKLING2_WEATHER_CALLS = [ + ("get_weather", _INKLING2_LISBON), + ("get_weather", _INKLING2_OSLO), +] + + +def _inkling2_request( + client, + text, + api="chat", + *, + stream=False, + chunk=0, + batched=True, + finish_reason="stop", + **payload, +): + """Answer one request with the raw model output ``text``. + + ``chunk`` is the size of the streamed pieces, so 1 splits every control + token and every JSON value. The finish reason arrives on a final empty + token, as it does after the end-of-turn token. Returns the response and + the prompts rendered by the real ``apply_chat_template``. + """ + size = chunk or len(text) + pieces = [text[i : i + size] for i in range(0, len(text), size)] + tokens = [*map(_token, pieces), _token("", finish_reason=finish_reason)] + tokenizer = fast_tokenizer( + chat_template=_INKLING2_CHAT_TEMPLATE, + response_template=_INKLING2_RESPONSE_TEMPLATE, + ) + prompts = [] + + def render(*args, **kwargs): + prompts.append(apply_chat_template(*args, **kwargs)) + return prompts[-1] + + with _endpoint( + config=NS(model_type="inkling2"), + processor=NS(tokenizer=tokenizer), + result=_result(text, finish_reason=finish_reason), + chunks=[_result(t.text, finish_reason=t.finish_reason) for t in tokens], + generator=_streaming(tokens) if batched else None, + ) as fake: + fake.template.side_effect = render + response = _post(client, api, stream=stream, **payload) + assert response.status_code == 200, response.text + return response, prompts + + +def _inkling2_chat(response, stream): + """Reduce a chat completion, streamed or not, to one comparable turn.""" + if stream: + events = _data(response) + assert not any("error" in event for event in events) + choices = [event["choices"][0] for event in events if event.get("choices")] + deltas = [choice["delta"] for choice in choices] + else: + choices = response.json()["choices"] + deltas = [choices[0]["message"]] + calls = {} + for delta in deltas: + for call in delta.get("tool_calls") or []: + function = call["function"] + entry = calls.setdefault(call["index"], dict(name="", arguments="")) + entry.update(id=call.get("id", entry.get("id"))) + entry["name"] += function.get("name") or "" + entry["arguments"] += function.get("arguments") or "" + assert list(calls) == list(range(len(calls))) + return NS( + reasoning=_joined(deltas, "reasoning_content"), + content=_joined(deltas, "content"), + calls=[(c["name"], json.loads(c["arguments"])) for c in calls.values()], + ids=[c["id"] for c in calls.values()], + finish=[c["finish_reason"] for c in choices if c.get("finish_reason")], + deltas=deltas, + ) + + +@pytest.mark.parametrize( + "stream,chunk,batched", + [ + (False, 0, True), + (False, 0, False), + (True, 0, True), + (True, 1, True), + (True, 1, False), + (True, 5, True), + (True, 11, False), + ], +) +def test_inkling2_chat_parallel_tool_calls(client, stream, chunk, batched): + response, _ = _inkling2_request( + client, + INKLING2_THINKING + _INKLING2_CALLS, + stream=stream, + chunk=chunk, + batched=batched, + tools=[_tool()], + ) + turn = _inkling2_chat(response, stream) + assert turn.calls == _INKLING2_WEATHER_CALLS + assert len(set(turn.ids)) == 2 and all(turn.ids) + assert turn.finish == ["tool_calls"] + assert turn.reasoning == "I need both cities." + assert turn.content == "" + # No control token, and no tool name outside the calls themselves. + assert "<|" not in response.text + assert response.text.count("get_weather") == 2 + if stream: + kinds = [ + "reasoning" if delta.get("reasoning_content") else "call" + for delta in turn.deltas + if delta.get("reasoning_content") or delta.get("tool_calls") + ] + assert kinds == sorted(kinds, reverse=True) and set(kinds) == { + "reasoning", + "call", + } + else: + message = response.json()["choices"][0]["message"] + assert message["content"] is None + assert all( + isinstance(call["function"]["arguments"], str) + for call in message["tool_calls"] + ) + + +@pytest.mark.parametrize("stream,chunk", [(False, 0), (True, 1), (True, 7)]) +@pytest.mark.parametrize( + "text,reasoning,content,calls", + [ + (_inkling2_call("get_time"), "", "", [("get_time", {})]), + ( + INKLING2_THINKING + INKLING2_TEXT + _inkling2_call("get_time"), + "I need both cities.", + "Lisbon is warmer.", + [("get_time", {})], + ), + ( + "<|message_model|><|content_text|>One. <|end_message|>" + + _inkling2_call("get_time") + + "<|message_model|><|content_tool_error|>Two.<|end_message|>", + "", + "One. Two.", + [("get_time", {})], + ), + ( + INKLING2_THINKING + INKLING2_TEXT, + "I need both cities.", + "Lisbon is warmer.", + [], + ), + ], + ids=["call-only", "text-then-call", "text-around-call", "answer-only"], +) +def test_inkling2_chat_turn_shapes( + client, stream, chunk, text, reasoning, content, calls +): + def turn(**payload): + response, _ = _inkling2_request( + client, text, stream=stream, chunk=chunk, **payload + ) + assert "<|" not in response.text + return _inkling2_chat(response, stream) + + with_tools = turn(tools=[_tool("get_time")]) + assert with_tools.calls == calls + assert with_tools.reasoning == reasoning + assert with_tools.content == content + assert with_tools.finish == ["tool_calls" if calls else "stop"] + # Without tools nothing is executable; reasoning and content are the same. + without_tools = turn() + assert without_tools.calls == [] + assert without_tools.finish == ["stop"] + assert without_tools.reasoning == reasoning + assert without_tools.content == content + + +@pytest.mark.parametrize("stream,chunk", [(False, 0), (True, 1), (True, 9)]) +def test_inkling2_chat_arguments_round_trip(client, stream, chunk): + text = INKLING2_THINKING + _inkling2_call("write_file", **INKLING2_ARGUMENTS) + response, _ = _inkling2_request( + client, text, stream=stream, chunk=chunk, tools=[_tool("write_file")] + ) + turn = _inkling2_chat(response, stream) + assert turn.calls == [("write_file", INKLING2_ARGUMENTS)] + assert turn.finish == ["tool_calls"] + assert turn.reasoning == "I need both cities." + assert turn.content == "" + + +@pytest.mark.parametrize("stream,chunk", [(False, 0), (True, 1), (True, 6)]) +@pytest.mark.parametrize("batched", [False, True]) +@pytest.mark.parametrize( + "text,finish_reason,calls,finish", + [ + # Cut by max_tokens inside the arguments: there is no call to run. + ( + INKLING2_THINKING + _inkling2_call("get_weather", city="Lisbon")[:-24], + "length", + [], + "length", + ), + # A call completed before the cut is still returned. + ( + _inkling2_call("get_time") + + _inkling2_call("get_weather", city="Lisbon")[:-24], + "length", + [("get_time", {})], + "tool_calls", + ), + ( + _inkling2_call("get_weather", raw='{"name":"get_weather","args":{city}}'), + "stop", + [], + "stop", + ), + ( + _inkling2_call("get_weather", raw='{"name":"get_weather"') + + _inkling2_call("get_time"), + "stop", + [("get_time", {})], + "tool_calls", + ), + ], + ids=["truncated", "truncated-after-call", "malformed", "malformed-then-call"], +) +def test_inkling2_chat_incomplete_calls( + client, stream, chunk, batched, text, finish_reason, calls, finish +): + response, _ = _inkling2_request( + client, + text, + stream=stream, + chunk=chunk, + batched=batched, + finish_reason=finish_reason, + tools=[_tool("get_weather"), _tool("get_time")], + ) + turn = _inkling2_chat(response, stream) + assert turn.calls == calls + assert turn.finish == [finish] + assert turn.content == "" + assert turn.reasoning == ("I need both cities." if "thinking" in text else "") + assert "<|" not in response.text and "city" not in response.text + + +@pytest.mark.parametrize("stream,chunk", [(False, 0), (True, 0), (True, 1)]) +def test_inkling2_messages_tool_use(client, stream, chunk): + response, _ = _inkling2_request( + client, + INKLING2_THINKING + INKLING2_TEXT + _INKLING2_CALLS, + "messages", + stream=stream, + chunk=chunk, + tools=[_tool(api="messages")], + thinking=dict(type="enabled", budget_tokens=64), + max_tokens=256, + ) + if stream: + events = _data(response) + blocks = [e["content_block"] for e in events if "content_block" in e] + inputs = [ + e["delta"]["partial_json"] + for e in events + if e.get("delta", {}).get("type") == "input_json_delta" + ] + names = [block["name"] for block in blocks if block["type"] == "tool_use"] + assert len(names) == len(inputs) + calls = list(zip(names, map(json.loads, inputs))) + thinking, text = _thinking_text(response, "messages") + stop_reason = _anthropic_stream_stop(response)[0] + else: + body = response.json() + blocks = body["content"] + calls = [(b["name"], b["input"]) for b in blocks if b["type"] == "tool_use"] + thinking = _joined(blocks, "thinking") + text = _joined(blocks, "text") + stop_reason = body["stop_reason"] + assert [block["type"] for block in blocks] == [ + "thinking", + "text", + "tool_use", + "tool_use", + ] + assert calls == _INKLING2_WEATHER_CALLS + assert len({b["id"] for b in blocks if b["type"] == "tool_use"}) == 2 + assert thinking == "I need both cities." + assert text == "Lisbon is warmer." + assert stop_reason == "tool_use" + assert "<|" not in response.text + + +@pytest.mark.parametrize("stream,chunk", [(False, 0), (True, 0), (True, 1)]) +def test_inkling2_responses_function_calls(client, stream, chunk): + server.response_store.clear() + server.response_store_order.clear() + response, _ = _inkling2_request( + client, + INKLING2_THINKING + INKLING2_TEXT + _INKLING2_CALLS, + "responses", + stream=stream, + chunk=chunk, + tools=[dict(type="function", name="get_weather", parameters={})], + ) + body = _completed_response(response) if stream else response.json() + items = {} + for item in body["output"]: + items.setdefault(item["type"], []).append(item) + calls = items.pop("function_call") + assert [ + (call["name"], json.loads(call["arguments"])) for call in calls + ] == _INKLING2_WEATHER_CALLS + assert len({call["call_id"] for call in calls}) == 2 + (reasoning,) = items.pop("reasoning") + assert reasoning["summary"][0]["text"] == "I need both cities." + # The text beside the calls is a message item only when streamed. + for message in items.pop("message", []): + assert message["content"][0]["text"] == "Lisbon is warmer." + assert not items + assert body["output_text"] == "Lisbon is warmer." + assert body["status"] == "completed" + assert "<|" not in response.text + + +@pytest.mark.parametrize("api", ["messages", "responses"]) +@pytest.mark.parametrize("stream", [False, True]) +def test_inkling2_truncated_call_ends_the_turn_cleanly(client, api, stream): + server.response_store.clear() + server.response_store_order.clear() + tool = ( + _tool(api="messages") + if api == "messages" + else dict(type="function", name="get_weather", parameters={}) + ) + response, _ = _inkling2_request( + client, + INKLING2_TEXT + _inkling2_call("get_weather", city="Lisbon")[:-24], + api, + stream=stream, + chunk=4 * stream, + finish_reason="length", + tools=[tool], + ) + if api == "responses": + body = _completed_response(response) if stream else response.json() + assert body["status"] == "incomplete" + assert [item["type"] for item in body["output"]] == ["message"] + assert body["output_text"] == "Lisbon is warmer." + elif stream: + assert _thinking_text(response, api) == ("", "Lisbon is warmer.") + assert _anthropic_stream_stop(response) == ("max_tokens", None) + else: + body = response.json() + assert body["content"] == [dict(type="text", text="Lisbon is warmer.")] + assert body["stop_reason"] == "max_tokens" + assert "<|" not in response.text and "city" not in response.text + + +_INKLING2_HISTORY = dict( + chat=dict( + messages=[ + _msg("Compare Lisbon and Oslo."), + _msg( + None, + "assistant", + reasoning_content="I need both cities.", + tool_calls=[ + dict( + id=f"call_{city}", + type="function", + # Clients send the arguments back as a JSON string. + function=dict( + name="get_weather", arguments=json.dumps(dict(city=city)) + ), + ) + for city in ("Lisbon", "Oslo") + ], + ), + _msg("19C", "tool", tool_call_id="call_Lisbon"), + _msg("3C", "tool", tool_call_id="call_Oslo"), + ], + tools=[_tool()], + ), + responses=dict( + input=[ + _input_message("Compare Lisbon and Oslo."), + *( + dict( + type="function_call", + name="get_weather", + arguments=json.dumps(dict(city=city)), + call_id=f"call_{city}", + ) + for city in ("Lisbon", "Oslo") + ), + dict(type="function_call_output", call_id="call_Lisbon", output="19C"), + dict(type="function_call_output", call_id="call_Oslo", output="3C"), + ], + tools=[dict(type="function", name="get_weather", parameters={})], + ), + messages=dict( + messages=[ + _msg("Compare Lisbon and Oslo."), + _msg( + [ + dict( + type="tool_use", + id=f"call_{city}", + name="get_weather", + input=dict(city=city), + ) + for city in ("Lisbon", "Oslo") + ], + "assistant", + ), + _msg( + [ + dict(type="tool_result", tool_use_id="call_Lisbon", content="19C"), + dict(type="tool_result", tool_use_id="call_Oslo", content="3C"), + ] + ), + ], + tools=[_tool(api="messages")], + max_tokens=64, + ), +) + + +@pytest.mark.parametrize("api", ["chat", "responses", "messages"]) +def test_inkling2_history_with_string_arguments_renders(client, api): + # The chat template refuses string arguments, so the stored history must + # reach it as objects, and each tool result must find its call's name. + _, prompts = _inkling2_request(client, INKLING2_TEXT, api, **_INKLING2_HISTORY[api]) + prompt = prompts[-1] + assert prompt.startswith("<|message_system|>tool_declare<|content_xml|>[{") + rendered = [ + _inkling2_call("get_weather", city="Lisbon"), + _inkling2_call("get_weather", city="Oslo"), + "<|message_tool|>get_weather<|content_text|>19C<|end_message|>", + "<|message_tool|>get_weather<|content_text|>3C<|end_message|>", + ] + assert all(prompt.count(part) == 1 for part in rendered) + assert sorted(rendered, key=prompt.index) == rendered + if api == "chat": + assert "<|content_thinking|>I need both cities.<|end_message|>" in prompt + + +@pytest.mark.parametrize("api", ["chat", "responses"]) +@pytest.mark.parametrize( + "effort,level", + [ + (None, "0.9"), + ("none", "0"), + ("minimal", "0.1"), + ("low", "0.2"), + ("medium", "0.7"), + ("high", "0.9"), + ("xhigh", "0.99"), + ("max", "0.99"), + ], +) +def test_inkling2_reasoning_effort_reaches_template(client, api, effort, level): + payload = {} + if effort is not None: + payload = ( + dict(reasoning_effort=effort) + if api == "chat" + else dict(reasoning=dict(effort=effort)) + ) + _, prompts = _inkling2_request(client, INKLING2_TEXT, api, **payload) + assert f"Thinking effort level: {level}<|end_message|>" in prompts[-1] + + +@pytest.mark.parametrize("stream", [False, True]) +def test_inkling2_split_survives_an_undecodable_call(stream): + # The response template decodes a call's JSON when its region closes, and + # raises on a truncated or malformed one. That must not cost the turn its + # reasoning and content, nor put the raw text into the content. + processor = NS( + tokenizer=fast_tokenizer(response_template=_INKLING2_RESPONSE_TEMPLATE) + ) + for call in ( + _inkling2_call("get_weather", city="Lisbon")[:-24], + _inkling2_call("get_weather", raw="{city}"), + _inkling2_call("get_weather", raw='{"name":"get_weather"}'), + ): + text = INKLING2_THINKING + call + INKLING2_TEXT + closed = call.endswith("<|end_message|>") + expected = ("I need both cities.", "Lisbon is warmer." if closed else "") + if stream: + state = server.make_response_stream_state(processor) + assert _thoughts(_feed_thinking(state, list(text), last=True)) == expected + else: + assert server._split_thinking(text, processor=processor) == expected + + def test_chat_completions_endpoint_falls_back_from_video_to_images(client): from mlx_vlm.generate import video diff --git a/mlx_vlm/tools/base.py b/mlx_vlm/tools/base.py index de8f5486..7432dcb5 100644 --- a/mlx_vlm/tools/base.py +++ b/mlx_vlm/tools/base.py @@ -6,6 +6,12 @@ string for formats whose call ends at a newline rather than a closing marker. The extractor (:mod:`mlx_vlm.tools.extractor`) drives any module shaped like this; the :class:`ParserSpec` in :mod:`mlx_vlm.tools.registry` maps a chat template to the module that owns it. + +A format whose calls two markers cannot delimit may also define +``find_tool_calls(text) -> [(start, end), ...]``. It receives the whole +assistant turn, reasoning included, and returns the span of every executable +call in order; the extractor then uses those spans instead of the markers +(see :mod:`mlx_vlm.tools.parsers.inkling2`). """ from __future__ import annotations diff --git a/mlx_vlm/tools/extractor.py b/mlx_vlm/tools/extractor.py index 2ff57b66..36b3bdd2 100644 --- a/mlx_vlm/tools/extractor.py +++ b/mlx_vlm/tools/extractor.py @@ -1,15 +1,16 @@ """Extract structured tool calls from a model's text output. Format-agnostic: given the parser module selected for the model (see -:mod:`mlx_vlm.tools.registry`), it slices the text between the parser's markers, -delegates per-format decoding to ``parse_tool_call``, and emits OpenAI-shaped -calls. +:mod:`mlx_vlm.tools.registry`), it slices the text between the parser's markers +(or at the spans the parser's own ``find_tool_calls`` reports), delegates +per-format decoding to ``parse_tool_call``, and emits OpenAI-shaped calls. """ import json import logging import re import uuid +from typing import List, Tuple from .base import ToolParser from .types import ParseResult @@ -17,6 +18,23 @@ from .types import ParseResult logger = logging.getLogger("mlx_vlm.server") +def _call_spans(model_output: str, tool_module: ToolParser) -> List[Tuple[int, int]]: + """Return the ``(start, end)`` span of every tool call, markers included.""" + find_tool_calls = getattr(tool_module, "find_tool_calls", None) + if find_tool_calls is not None: + return find_tool_calls(model_output) + if tool_module.tool_call_end == "": + pattern = re.compile( + f"{re.escape(tool_module.tool_call_start)}.*?(?:\n|$)", re.DOTALL + ) + else: + pattern = re.compile( + f"{re.escape(tool_module.tool_call_start)}.*?{re.escape(tool_module.tool_call_end)}", + re.DOTALL, + ) + return [match.span() for match in pattern.finditer(model_output)] + + def process_tool_calls( model_output: str, tool_module: ToolParser, tools ) -> ParseResult: @@ -25,19 +43,14 @@ def process_tool_calls( remaining = model_output if tool_module.tool_call_start in model_output: - if tool_module.tool_call_end == "": - pattern = re.compile( - f"{re.escape(tool_module.tool_call_start)}.*?(?:\n|$)", re.DOTALL - ) - else: - pattern = re.compile( - f"{re.escape(tool_module.tool_call_start)}.*?{re.escape(tool_module.tool_call_end)}", - re.DOTALL, - ) - - matches = re.findall(pattern, model_output) - if matches: - remaining = re.sub(pattern, " ", model_output).strip() + spans = _call_spans(model_output, tool_module) + if spans: + matches = [model_output[start:end] for start, end in spans] + # Each call is replaced with a space. + edges = [0, *(edge for span in spans for edge in span), len(model_output)] + remaining = " ".join( + model_output[start:end] for start, end in zip(edges[::2], edges[1::2]) + ).strip() for match in matches: call = ( match.strip() diff --git a/mlx_vlm/tools/parsers/inkling2.py b/mlx_vlm/tools/parsers/inkling2.py new file mode 100644 index 00000000..1f9d18f0 --- /dev/null +++ b/mlx_vlm/tools/parsers/inkling2.py @@ -0,0 +1,99 @@ +"""Inkling 2 tool calls. + +Inkling 2 writes an assistant turn as a run of messages, each opened by a role +header and a content-type token and closed by ``<|end_message|>``:: + + <|message_model|><|content_thinking|>I need the weather.<|end_message|> + <|message_model|><|content_text|>Let me check.<|end_message|> + <|message_model|>get_weather<|content_invoke_tool_json|>{"name":"get_weather","args":{"city":"Oslo"}}<|end_message|> + +A tool call is a message of type ``invoke_tool_json``: its header repeats the +tool name and its body is one JSON object with ``name`` and ``args``. Parallel +calls are consecutive call messages. + +The two markers alone do not delimit a call: the header belongs to it, and the +closing token also closes every other message. :func:`find_tool_calls` therefore +walks the turn message by message, the way the tokenizer's ``response_template`` +does, so call markup quoted inside a reasoning or text message is not a call. + +A call whose body is not a JSON object with a string ``name`` and an object +``args`` is rejected: the extractor logs and drops it, and keeps the calls +around it. A call still open when generation stops (``max_tokens``) is not a +call. + +Limit: this works on decoded text, where a control token and the same characters +written as ordinary text look alike. A call body is JSON, so ``<|end_message|>`` +inside one of its strings is recognised as data (see :func:`_call_end`). In a +reasoning or text message those characters end the message. The server splits +reasoning and content with the response template, which does not read JSON: a +message opening such as ``<|content_text|>`` written inside an argument string +after those characters also shows up as content. +""" + +import json +import re +from typing import Any, List, Optional, Tuple + +tool_call_start = "<|content_invoke_tool_json|>" +tool_call_end = "<|end_message|>" + +_HEADER = r"(?:<\|message_model\|>[^<]*)?" +# The message openings the response template knows; other text is not a message. +_MESSAGE = re.compile( + _HEADER + + r"<\|content_(thinking|text|tool_error|invoke_tool_text|invoke_tool_json)\|>" +) +_CALL_OPENING = re.compile(rf"\s*{_HEADER}{re.escape(tool_call_start)}") + + +def _call_end(text: str, start: int) -> int: + """Index of the ``<|end_message|>`` that closes the call body at ``start``. + + The call ends at the first marker that leaves valid JSON before it, so the + marker's characters inside a JSON string stay part of the arguments. A body + that is valid at no marker ends at the first one, where + :func:`parse_tool_call` rejects it. Returns -1 while the call is unclosed. + """ + first = end = text.find(tool_call_end, start) + while end >= 0: + try: + json.loads(text[start:end]) + return end + except ValueError: + end = text.find(tool_call_end, end + len(tool_call_end)) + return first + + +def find_tool_calls(text: str) -> List[Tuple[int, int]]: + """Return the ``(start, end)`` span of every call message, header included. + + A message still open when the text ends (generation was cut off) is not a + call, and nothing after it is. + """ + spans = [] + position = 0 + while (message := _MESSAGE.search(text, position)) is not None: + is_call = message[1] == "invoke_tool_json" + end = ( + _call_end(text, message.end()) + if is_call + else text.find(tool_call_end, message.end()) + ) + if end < 0: + break + position = end + len(tool_call_end) + if is_call: + spans.append((message.start(), position)) + return spans + + +def parse_tool_call(text: str, tools: Optional[Any] = None) -> dict: + """Decode one call body, with or without its ``<|message_model|>NAME`` opening.""" + opening = _CALL_OPENING.match(text) + call = json.loads(text[opening.end() :] if opening else text) + if not isinstance(call, dict) or not isinstance(call.get("name"), str): + raise ValueError("Inkling 2 tool call must be an object with a name.") + arguments = call.get("args", {}) + if not isinstance(arguments, dict): + raise ValueError("Inkling 2 tool call args must be an object.") + return {"name": call["name"], "arguments": arguments} diff --git a/mlx_vlm/tools/registry.py b/mlx_vlm/tools/registry.py index ddf3b278..8f4447f7 100644 --- a/mlx_vlm/tools/registry.py +++ b/mlx_vlm/tools/registry.py @@ -65,6 +65,7 @@ SPECS: Tuple[ParserSpec, ...] = ( ParserSpec("kimi_k2", (("<|tool_calls_section_begin|>",),)), ParserSpec("mistral", (("[TOOL_CALLS]",),)), ParserSpec("harmony", (("<|channel|>commentary", "<|call|>"),)), + ParserSpec("inkling2", (("<|content_invoke_tool_json|>",),)), ParserSpec("json_tools", (("<tool_call>", "tool_call.name"),), priority=0), )
diff --git a/mlx_vlm/server/anthropic.py b/mlx_vlm/server/anthropic.py
index ed46ff05..1b714bec 100644
--- a/mlx_vlm/server/anthropic.py
+++ b/mlx_vlm/server/anthropic.py
@@ -29,11 +29,11 @@ from .generation import (
from .responses_state import (
ThinkingStreamState,
ToolCallStreamState,
+ content_beside_tool_calls,
finish_content_streams,
make_response_stream_state,
parse_assistant_tool_calls,
prompt_has_open_thinking,
- strip_protocol_markers,
)
from .runtime import runtime
from .schemas import AnthropicMessageResponse, AnthropicRequest, AnthropicUsage
@@ -1130,20 +1130,10 @@ async def anthropic_messages_endpoint(http_request: Request):
tc = parse_tools(full_text)
if tc.calls:
parsed_tool_calls = tc.calls
- # Structured templates already removed tool/routing regions
- # from the content parsed above. Otherwise use the delimiter
- # parser's remainder (reasoning has already been excluded).
- if not (
- getattr(processor, "response_template", None)
- or getattr(
- getattr(processor, "tokenizer", None),
- "response_template",
- None,
- )
- ):
- content = tc.remaining_text or ""
- content = strip_protocol_markers(
+ content = content_beside_tool_calls(
content,
+ tc.remaining_text,
+ processor,
tool_module,
gen_args.thinking_start_token,
gen_args.thinking_end_token,
diff --git a/mlx_vlm/server/openai.py b/mlx_vlm/server/openai.py
index a1729e13..3d4d0048 100644
--- a/mlx_vlm/server/openai.py
+++ b/mlx_vlm/server/openai.py
@@ -60,13 +60,13 @@ from .responses_state import (
from .responses_state import _sse_event as _response_sse_event
from .responses_state import (
_store_response,
+ content_beside_tool_calls,
finish_content_streams,
make_response_stream_state,
parse_assistant_tool_calls,
prompt_has_open_thinking,
response_store,
response_store_lock,
- strip_protocol_markers,
)
from .runtime import runtime
from .schemas import (
@@ -2807,11 +2807,11 @@ async def chat_completions_endpoint(request: ChatRequest, http_request: Request)
tc = parse_tools(full_text)
if tc.calls:
parsed_tool_calls = tc.calls
- # Reasoning was already removed before tool parsing.
- clean_remaining = tc.remaining_text or ""
content = (
- strip_protocol_markers(
- clean_remaining,
+ content_beside_tool_calls(
+ content,
+ tc.remaining_text,
+ processor,
tool_module,
gen_args.thinking_start_token,
gen_args.thinking_end_token,
diff --git a/mlx_vlm/server/responses_state.py b/mlx_vlm/server/responses_state.py
index d975a822..424a54ea 100644
--- a/mlx_vlm/server/responses_state.py
+++ b/mlx_vlm/server/responses_state.py
@@ -9,6 +9,7 @@ from threading import Lock
from typing import Any, Dict, List, Optional, Tuple
from fastapi import HTTPException
+from transformers.utils.chat_parsing import ResponseParser, parse_response
from transformers.utils.chat_parsing.response_templates import load_response_template
from ..prompt_utils import _normalize_tool_message
@@ -241,6 +242,29 @@ def _response_template_prefix(tokenizer, prompt: str) -> str:
return prompt if template.start_anchor_re.search(prompt) else ""
+_TEXT_FIELDS = ("content", "reasoning", "reasoning_content")
+_REGION_KEYS = ("open", "open_pattern", "close", "close_pattern", "repeats")
+
+
+def _text_response_template(template: Dict[str, Any]) -> Dict[str, Any]:
+ """Copy a response template, reading its structured fields as plain text.
+
+ The server takes reasoning and content from a response template; tool calls