File size: 7,421 Bytes
e5d5367
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
import json
import re
import uuid
from collections.abc import Sequence
from typing import Any, Optional

# --- Imports compatibili con vLLM vecchi (<= ~0.11) e nuovi (0.12+ / 0.19.x) ---
try:  # vLLM nuovi: protocol diviso per endpoint
    from vllm.entrypoints.openai.chat_completion.protocol import (
        ChatCompletionRequest,
    )
    from vllm.entrypoints.openai.engine.protocol import (
        DeltaFunctionCall,
        DeltaMessage,
        DeltaToolCall,
        ExtractedToolCallInformation,
        FunctionCall,
        ToolCall,
    )
except ImportError:  # vLLM vecchi: protocol unico
    from vllm.entrypoints.openai.protocol import (
        ChatCompletionRequest,
        DeltaFunctionCall,
        DeltaMessage,
        DeltaToolCall,
        ExtractedToolCallInformation,
        FunctionCall,
        ToolCall,
    )

try:  # vLLM nuovi: tool parser spostati in vllm.tool_parsers
    from vllm.tool_parsers.abstract_tool_parser import (
        ToolParser,
        ToolParserManager,
    )
except ImportError:  # vLLM vecchi
    from vllm.entrypoints.openai.tool_parsers.abstract_tool_parser import (
        ToolParser,
        ToolParserManager,
    )

try:
    from vllm.utils import random_uuid
except ImportError:  # vllm.utils riorganizzato in alcune versioni
    def random_uuid() -> str:
        return uuid.uuid4().hex


START_TAG = "<tool_call>"
END_TAG = "</tool_call>"


def _parse_calls(payload: str, request: ChatCompletionRequest) -> Optional[list]:
    """Return the calls in a ``<tool_call>`` payload, or None if invalid.

    Every call must name a function offered in ``request.tools`` and carry
    its arguments as a JSON object.
    """
    allowed_names = {tool.function.name for tool in request.tools or []}
    try:
        parsed = json.loads(payload)
    except ValueError:
        return None
    calls = parsed if isinstance(parsed, list) else [parsed]
    if not calls or not all(
        isinstance(call, dict)
        and isinstance(call.get("name"), str)
        and call["name"] in allowed_names
        and isinstance(call.get("arguments", {}), dict)
        for call in calls
    ):
        return None
    return [
        {"name": call["name"], "arguments": call.get("arguments", {})}
        for call in calls
    ]


def _partial_tag_len(text: str) -> int:
    """Length of the longest suffix of ``text`` that starts ``START_TAG``."""
    for size in range(min(len(text), len(START_TAG) - 1), 0, -1):
        if START_TAG.startswith(text[-size:]):
            return size
    return 0


@ToolParserManager.register_module("velvet")
class VelvetToolParser(ToolParser):
    """Parse Velvet ``<tool_call>[...]</tool_call>`` output."""

    _pattern = re.compile(r"<tool_call>\s*(.*?)\s*</tool_call>", re.DOTALL)

    def __init__(self, tokenizer: Any, *args: Any, **kwargs: Any):
        # *args/**kwargs: inoltra eventuali parametri aggiuntivi che le
        # versioni più recenti di vLLM passano al costruttore.
        super().__init__(tokenizer, *args, **kwargs)
        # Characters of the generated text already streamed or consumed.
        self._scan_pos = 0

    def adjust_request(
        self, request: ChatCompletionRequest
    ) -> ChatCompletionRequest:
        # <tool_call> and </tool_call> are special tokens: keep them in the
        # decoded text, otherwise the parser cannot find the call.
        if request.tools and request.tool_choice != "none":
            request.skip_special_tokens = False
        return request

    def extract_tool_calls(
        self, model_output: str, request: ChatCompletionRequest
    ) -> ExtractedToolCallInformation:
        match = self._pattern.search(model_output)
        calls = _parse_calls(match.group(1), request) if match else None
        if calls is None:
            return ExtractedToolCallInformation(
                tools_called=False, tool_calls=[], content=model_output
            )

        tool_calls = [
            ToolCall(
                type="function",
                function=FunctionCall(
                    name=call["name"],
                    arguments=json.dumps(call["arguments"], ensure_ascii=False),
                ),
            )
            for call in calls
        ]
        prefix = model_output[: match.start()].strip()
        return ExtractedToolCallInformation(
            tools_called=True, tool_calls=tool_calls, content=prefix or None
        )

    def extract_tool_calls_streaming(
        self,
        previous_text: str,
        current_text: str,
        delta_text: str,
        previous_token_ids: Sequence[int],
        current_token_ids: Sequence[int],
        delta_token_ids: Sequence[int],
        request: ChatCompletionRequest,
    ) -> DeltaMessage:
        """Stream plain text as it arrives; send each tool call whole.

        Text inside ``<tool_call>...</tool_call>`` is held back until the
        closing tag arrives. A valid block becomes one chunk of tool calls;
        an invalid one is released as plain text, as in non-streaming mode.
        """
        content = ""
        new_calls = []
        pos = self._scan_pos

        while True:
            start = current_text.find(START_TAG, pos)
            if start == -1:
                hold = _partial_tag_len(current_text[pos:])
                content += current_text[pos : len(current_text) - hold]
                pos = len(current_text) - hold
                break
            content += current_text[pos:start]
            pos = start
            end = current_text.find(END_TAG, start)
            if end == -1:
                break  # wait for the closing tag
            block_end = end + len(END_TAG)
            calls = _parse_calls(
                current_text[start + len(START_TAG) : end].strip(), request
            )
            if calls is None:
                content += current_text[start:block_end]
            else:
                new_calls.extend(calls)
            pos = block_end

        # Release anything still held back when generation stops.
        max_tokens = getattr(request, "max_completion_tokens", None) or getattr(
            request, "max_tokens", None
        )
        eos_id = getattr(self.model_tokenizer, "eos_token_id", None)
        finished = (eos_id is not None and eos_id in delta_token_ids) or (
            max_tokens is not None and len(current_token_ids) >= max_tokens
        )
        if finished:
            content += current_text[pos:]
            pos = len(current_text)
        self._scan_pos = pos

        delta_calls = []
        for call in new_calls:
            arguments = json.dumps(call["arguments"], ensure_ascii=False)
            self.current_tool_id += 1
            self.prev_tool_call_arr.append(call)
            self.streamed_args_for_tool.append(arguments)
            delta_calls.append(
                DeltaToolCall(
                    index=self.current_tool_id,
                    id=f"chatcmpl-tool-{random_uuid()}",
                    type="function",
                    function=DeltaFunctionCall(
                        name=call["name"], arguments=arguments
                    ).model_dump(exclude_none=True),
                )
            )

        # An empty delta (rather than None) keeps vLLM sending the final
        # chunk with finish_reason while a block is still buffered.
        return DeltaMessage(content=content or None, tool_calls=delta_calls)