Velvet2-3B-MLX-8bit / tool_parser_plugin.py
rbizzoni's picture
Upload folder using huggingface_hub
e5d5367 verified
Raw History Blame Contribute Delete
7.42 kB
import json
import re
import uuid
from collections.abc import Sequence
from typing import Any, Optional
# --- Imports compatibili con vLLM vecchi (<= ~0.11) e nuovi (0.12+ / 0.19.x) ---
try: # vLLM nuovi: protocol diviso per endpoint
from vllm.entrypoints.openai.chat_completion.protocol import (
ChatCompletionRequest,
)
from vllm.entrypoints.openai.engine.protocol import (
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
ExtractedToolCallInformation,
FunctionCall,
ToolCall,
)
except ImportError: # vLLM vecchi: protocol unico
from vllm.entrypoints.openai.protocol import (
ChatCompletionRequest,
DeltaFunctionCall,
DeltaMessage,
DeltaToolCall,
ExtractedToolCallInformation,
FunctionCall,
ToolCall,
)
try: # vLLM nuovi: tool parser spostati in vllm.tool_parsers
from vllm.tool_parsers.abstract_tool_parser import (
ToolParser,
ToolParserManager,
)
except ImportError: # vLLM vecchi
from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
ToolParser,
ToolParserManager,
)
try:
from vllm.utils import random_uuid
except ImportError: # vllm.utils riorganizzato in alcune versioni
def random_uuid() -> str:
return uuid.uuid4().hex
START_TAG = "<tool_call>"
END_TAG = "</tool_call>"
def _parse_calls(payload: str, request: ChatCompletionRequest) -> Optional[list]:
"""Return the calls in a ``<tool_call>`` payload, or None if invalid.
Every call must name a function offered in ``request.tools`` and carry
its arguments as a JSON object.
"""
allowed_names = {tool.function.name for tool in request.tools or []}
try:
parsed = json.loads(payload)
except ValueError:
return None
calls = parsed if isinstance(parsed, list) else [parsed]
if not calls or not all(
isinstance(call, dict)
and isinstance(call.get("name"), str)
and call["name"] in allowed_names
and isinstance(call.get("arguments", {}), dict)
for call in calls
):
return None
return [
{"name": call["name"], "arguments": call.get("arguments", {})}
for call in calls
]
def _partial_tag_len(text: str) -> int:
"""Length of the longest suffix of ``text`` that starts ``START_TAG``."""
for size in range(min(len(text), len(START_TAG) - 1), 0, -1):
if START_TAG.startswith(text[-size:]):
return size
return 0
@ToolParserManager.register_module("velvet")
class VelvetToolParser(ToolParser):
"""Parse Velvet ``<tool_call>[...]</tool_call>`` output."""
_pattern = re.compile(r"<tool_call>\s*(.*?)\s*</tool_call>", re.DOTALL)
def __init__(self, tokenizer: Any, *args: Any, **kwargs: Any):
# *args/**kwargs: inoltra eventuali parametri aggiuntivi che le
# versioni più recenti di vLLM passano al costruttore.
super().__init__(tokenizer, *args, **kwargs)
# Characters of the generated text already streamed or consumed.
self._scan_pos = 0
def adjust_request(
self, request: ChatCompletionRequest
) -> ChatCompletionRequest:
# <tool_call> and </tool_call> are special tokens: keep them in the
# decoded text, otherwise the parser cannot find the call.
if request.tools and request.tool_choice != "none":
request.skip_special_tokens = False
return request
def extract_tool_calls(
self, model_output: str, request: ChatCompletionRequest
) -> ExtractedToolCallInformation:
match = self._pattern.search(model_output)
calls = _parse_calls(match.group(1), request) if match else None
if calls is None:
return ExtractedToolCallInformation(
tools_called=False, tool_calls=[], content=model_output
)
tool_calls = [
ToolCall(
type="function",
function=FunctionCall(
name=call["name"],
arguments=json.dumps(call["arguments"], ensure_ascii=False),
),
)
for call in calls
]
prefix = model_output[: match.start()].strip()
return ExtractedToolCallInformation(
tools_called=True, tool_calls=tool_calls, content=prefix or None
)
def extract_tool_calls_streaming(
self,
previous_text: str,
current_text: str,
delta_text: str,
previous_token_ids: Sequence[int],
current_token_ids: Sequence[int],
delta_token_ids: Sequence[int],
request: ChatCompletionRequest,
) -> DeltaMessage:
"""Stream plain text as it arrives; send each tool call whole.
Text inside ``<tool_call>...</tool_call>`` is held back until the
closing tag arrives. A valid block becomes one chunk of tool calls;
an invalid one is released as plain text, as in non-streaming mode.
"""
content = ""
new_calls = []
pos = self._scan_pos
while True:
start = current_text.find(START_TAG, pos)
if start == -1:
hold = _partial_tag_len(current_text[pos:])
content += current_text[pos : len(current_text) - hold]
pos = len(current_text) - hold
break
content += current_text[pos:start]
pos = start
end = current_text.find(END_TAG, start)
if end == -1:
break # wait for the closing tag
block_end = end + len(END_TAG)
calls = _parse_calls(
current_text[start + len(START_TAG) : end].strip(), request
)
if calls is None:
content += current_text[start:block_end]
else:
new_calls.extend(calls)
pos = block_end
# Release anything still held back when generation stops.
max_tokens = getattr(request, "max_completion_tokens", None) or getattr(
request, "max_tokens", None
)
eos_id = getattr(self.model_tokenizer, "eos_token_id", None)
finished = (eos_id is not None and eos_id in delta_token_ids) or (
max_tokens is not None and len(current_token_ids) >= max_tokens
)
if finished:
content += current_text[pos:]
pos = len(current_text)
self._scan_pos = pos
delta_calls = []
for call in new_calls:
arguments = json.dumps(call["arguments"], ensure_ascii=False)
self.current_tool_id += 1
self.prev_tool_call_arr.append(call)
self.streamed_args_for_tool.append(arguments)
delta_calls.append(
DeltaToolCall(
index=self.current_tool_id,
id=f"chatcmpl-tool-{random_uuid()}",
type="function",
function=DeltaFunctionCall(
name=call["name"], arguments=arguments
).model_dump(exclude_none=True),
)
)
# An empty delta (rather than None) keeps vLLM sending the final
# chunk with finish_reason while a block is still buffered.
return DeltaMessage(content=content or None, tool_calls=delta_calls)