import json import re import uuid from collections.abc import Sequence from typing import Any, Optional # --- Imports compatibili con vLLM vecchi (<= ~0.11) e nuovi (0.12+ / 0.19.x) --- try: # vLLM nuovi: protocol diviso per endpoint from vllm.entrypoints.openai.chat_completion.protocol import ( ChatCompletionRequest, ) from vllm.entrypoints.openai.engine.protocol import ( DeltaFunctionCall, DeltaMessage, DeltaToolCall, ExtractedToolCallInformation, FunctionCall, ToolCall, ) except ImportError: # vLLM vecchi: protocol unico from vllm.entrypoints.openai.protocol import ( ChatCompletionRequest, DeltaFunctionCall, DeltaMessage, DeltaToolCall, ExtractedToolCallInformation, FunctionCall, ToolCall, ) try: # vLLM nuovi: tool parser spostati in vllm.tool_parsers from vllm.tool_parsers.abstract_tool_parser import ( ToolParser, ToolParserManager, ) except ImportError: # vLLM vecchi from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import ( ToolParser, ToolParserManager, ) try: from vllm.utils import random_uuid except ImportError: # vllm.utils riorganizzato in alcune versioni def random_uuid() -> str: return uuid.uuid4().hex START_TAG = "" END_TAG = "" def _parse_calls(payload: str, request: ChatCompletionRequest) -> Optional[list]: """Return the calls in a ```` payload, or None if invalid. Every call must name a function offered in ``request.tools`` and carry its arguments as a JSON object. """ allowed_names = {tool.function.name for tool in request.tools or []} try: parsed = json.loads(payload) except ValueError: return None calls = parsed if isinstance(parsed, list) else [parsed] if not calls or not all( isinstance(call, dict) and isinstance(call.get("name"), str) and call["name"] in allowed_names and isinstance(call.get("arguments", {}), dict) for call in calls ): return None return [ {"name": call["name"], "arguments": call.get("arguments", {})} for call in calls ] def _partial_tag_len(text: str) -> int: """Length of the longest suffix of ``text`` that starts ``START_TAG``.""" for size in range(min(len(text), len(START_TAG) - 1), 0, -1): if START_TAG.startswith(text[-size:]): return size return 0 @ToolParserManager.register_module("velvet") class VelvetToolParser(ToolParser): """Parse Velvet ``[...]`` output.""" _pattern = re.compile(r"\s*(.*?)\s*", re.DOTALL) def __init__(self, tokenizer: Any, *args: Any, **kwargs: Any): # *args/**kwargs: inoltra eventuali parametri aggiuntivi che le # versioni più recenti di vLLM passano al costruttore. super().__init__(tokenizer, *args, **kwargs) # Characters of the generated text already streamed or consumed. self._scan_pos = 0 def adjust_request( self, request: ChatCompletionRequest ) -> ChatCompletionRequest: # and are special tokens: keep them in the # decoded text, otherwise the parser cannot find the call. if request.tools and request.tool_choice != "none": request.skip_special_tokens = False return request def extract_tool_calls( self, model_output: str, request: ChatCompletionRequest ) -> ExtractedToolCallInformation: match = self._pattern.search(model_output) calls = _parse_calls(match.group(1), request) if match else None if calls is None: return ExtractedToolCallInformation( tools_called=False, tool_calls=[], content=model_output ) tool_calls = [ ToolCall( type="function", function=FunctionCall( name=call["name"], arguments=json.dumps(call["arguments"], ensure_ascii=False), ), ) for call in calls ] prefix = model_output[: match.start()].strip() return ExtractedToolCallInformation( tools_called=True, tool_calls=tool_calls, content=prefix or None ) def extract_tool_calls_streaming( self, previous_text: str, current_text: str, delta_text: str, previous_token_ids: Sequence[int], current_token_ids: Sequence[int], delta_token_ids: Sequence[int], request: ChatCompletionRequest, ) -> DeltaMessage: """Stream plain text as it arrives; send each tool call whole. Text inside ``...`` is held back until the closing tag arrives. A valid block becomes one chunk of tool calls; an invalid one is released as plain text, as in non-streaming mode. """ content = "" new_calls = [] pos = self._scan_pos while True: start = current_text.find(START_TAG, pos) if start == -1: hold = _partial_tag_len(current_text[pos:]) content += current_text[pos : len(current_text) - hold] pos = len(current_text) - hold break content += current_text[pos:start] pos = start end = current_text.find(END_TAG, start) if end == -1: break # wait for the closing tag block_end = end + len(END_TAG) calls = _parse_calls( current_text[start + len(START_TAG) : end].strip(), request ) if calls is None: content += current_text[start:block_end] else: new_calls.extend(calls) pos = block_end # Release anything still held back when generation stops. max_tokens = getattr(request, "max_completion_tokens", None) or getattr( request, "max_tokens", None ) eos_id = getattr(self.model_tokenizer, "eos_token_id", None) finished = (eos_id is not None and eos_id in delta_token_ids) or ( max_tokens is not None and len(current_token_ids) >= max_tokens ) if finished: content += current_text[pos:] pos = len(current_text) self._scan_pos = pos delta_calls = [] for call in new_calls: arguments = json.dumps(call["arguments"], ensure_ascii=False) self.current_tool_id += 1 self.prev_tool_call_arr.append(call) self.streamed_args_for_tool.append(arguments) delta_calls.append( DeltaToolCall( index=self.current_tool_id, id=f"chatcmpl-tool-{random_uuid()}", type="function", function=DeltaFunctionCall( name=call["name"], arguments=arguments ).model_dump(exclude_none=True), ) ) # An empty delta (rather than None) keeps vLLM sending the final # chunk with finish_reason while a block is still buffered. return DeltaMessage(content=content or None, tool_calls=delta_calls)