luclima commited on
Commit
2875c44
·
verified ·
1 Parent(s): dc89b9e

Upload folder using huggingface_hub (part 53)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. asian_ai/performance/vLLM/SECURITY.md +59 -0
  2. asian_ai/performance/vLLM/rust-toolchain.toml +2 -0
  3. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_hermes.jinja +130 -0
  4. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_hunyuan_a13b.jinja +113 -0
  5. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_internlm2_tool.jinja +60 -0
  6. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_llama3.1_json.jinja +120 -0
  7. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_llama3.2_json.jinja +133 -0
  8. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_llama3.2_pythonic.jinja +98 -0
  9. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_llama4_json.jinja +116 -0
  10. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_llama4_pythonic.jinja +111 -0
  11. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_mistral.jinja +86 -0
  12. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_mistral3.jinja +126 -0
  13. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_mistral_parallel.jinja +93 -0
  14. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_phi4_mini.jinja +62 -0
  15. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_qwen3coder.jinja +117 -0
  16. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_toolace.jinja +65 -0
  17. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_xlam_llama.jinja +77 -0
  18. asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_xlam_qwen.jinja +66 -0
  19. asian_ai/performance/vLLM/rust/src/cmd/Cargo.toml +41 -0
  20. asian_ai/performance/vLLM/rust/src/cmd/examples/README.md +44 -0
  21. asian_ai/performance/vLLM/rust/src/cmd/src/cli.rs +694 -0
  22. asian_ai/performance/vLLM/rust/src/cmd/src/cli/tests.rs +1718 -0
  23. asian_ai/performance/vLLM/rust/src/cmd/src/cli/unsupported.rs +578 -0
  24. asian_ai/performance/vLLM/rust/src/cmd/src/logging.rs +328 -0
  25. asian_ai/performance/vLLM/rust/src/cmd/src/main.rs +192 -0
  26. asian_ai/performance/vLLM/rust/src/engine-core-client/Cargo.toml +49 -0
  27. asian_ai/performance/vLLM/rust/src/engine-core-client/examples/README.md +53 -0
  28. asian_ai/performance/vLLM/rust/src/engine-core-client/examples/external_engine_logprobs.rs +190 -0
  29. asian_ai/performance/vLLM/rust/src/engine-core-client/examples/external_engine_utility_call.rs +144 -0
  30. asian_ai/performance/vLLM/rust/src/engine-core-client/src/client.rs +793 -0
  31. asian_ai/performance/vLLM/rust/src/engine-core-client/src/client/imp.rs +476 -0
  32. asian_ai/performance/vLLM/rust/src/engine-core-client/src/client/state.rs +926 -0
  33. asian_ai/performance/vLLM/rust/src/engine-core-client/src/client/stream.rs +153 -0
  34. asian_ai/performance/vLLM/rust/src/engine-core-client/src/coordinator/bootstrap.rs +91 -0
  35. asian_ai/performance/vLLM/rust/src/engine-core-client/src/coordinator/external.rs +167 -0
  36. asian_ai/performance/vLLM/rust/src/engine-core-client/src/coordinator/handle.rs +144 -0
  37. asian_ai/performance/vLLM/rust/src/engine-core-client/src/coordinator/inproc.rs +257 -0
  38. asian_ai/performance/vLLM/rust/src/engine-core-client/src/coordinator/mod.rs +7 -0
  39. asian_ai/performance/vLLM/rust/src/engine-core-client/src/error.rs +96 -0
  40. asian_ai/performance/vLLM/rust/src/engine-core-client/src/lib.rs +20 -0
  41. asian_ai/performance/vLLM/rust/src/engine-core-client/src/metrics.rs +259 -0
  42. asian_ai/performance/vLLM/rust/src/engine-core-client/src/mock_engine.rs +274 -0
  43. asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/dtype.rs +40 -0
  44. asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/handshake.rs +81 -0
  45. asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/logprobs.rs +285 -0
  46. asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/logprobs/array.rs +306 -0
  47. asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/logprobs/tests.rs +302 -0
  48. asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/logprobs/wire.rs +31 -0
  49. asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/lora.rs +42 -0
  50. asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/mod.rs +89 -0
asian_ai/performance/vLLM/SECURITY.md ADDED
@@ -0,0 +1,59 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Security Policy
2
+
3
+ ## Reporting security issues
4
+
5
+ Please report security issues privately using [the vulnerability submission form](https://github.com/vllm-project/vllm/security/advisories/new).
6
+
7
+ ## Issue triage
8
+
9
+ Reports will then be triaged by the [vulnerability management team](https://docs.vllm.ai/en/latest/contributing/vulnerability_management.html).
10
+
11
+ ## Threat model
12
+
13
+ Please see the [Security Guide in the vLLM documentation](https://docs.vllm.ai/en/latest/usage/security.html) for more information on vLLM's security assumptions and recommendations.
14
+
15
+ Please see [PyTorch's Security Policy](https://github.com/pytorch/pytorch/blob/main/SECURITY.md) for more information and recommendations on how to securely interact with models.
16
+
17
+ ## Issue severity
18
+
19
+ We will determine the risk of each issue, taking into account our experience dealing with past issues, versions affected, common defaults, and use cases. We use the following severity categories:
20
+
21
+ ### CRITICAL Severity
22
+
23
+ Vulnerabilities that allow remote attackers to execute arbitrary code, take full control of the system, or significantly compromise confidentiality, integrity, or availability without any interaction or privileges needed, examples include remote code execution via network, deserialization issues that allow exploit chains. Generally those issues which are rated as CVSS ≥ 9.0.
24
+
25
+ ### HIGH Severity
26
+
27
+ Serious security flaws that allow elevated impact—like RCE in specific, limited contexts or significant data loss—but require advanced conditions or some trust, examples include RCE in advanced deployment modes (e.g. multi-node), or high impact issues where some sort of privileged network access is required. These issues typically have CVSS scores between 7.0 and 8.9
28
+
29
+ ### MODERATE Severity
30
+
31
+ Vulnerabilities that cause denial of service or partial disruption, but do not allow arbitrary code execution or data breach and have limited impact. These issues have a CVSS rating between 4.0 and 6.9
32
+
33
+ ### LOW Severity
34
+
35
+ Minor issues such as informational disclosures, logging errors, non-exploitable flaws, or weaknesses that require local or high-privilege access and offer negligible impact. Examples include side channel attacks or hash collisions. These issues often have CVSS scores less than 4.0
36
+
37
+ ## Fix disclosure policy
38
+
39
+ When a security report is accepted, the fix process depends on the severity:
40
+
41
+ * **CRITICAL and HIGH severity**: Fixes are developed in a private security fork and coordinated with the prenotification group before public disclosure.
42
+ * **MODERATE and LOW severity**: Fixes are developed and submitted as public pull requests. These issues do not require embargo since they do not enable arbitrary code execution or significant data breach, and public visibility accelerates community review and adoption of the fix.
43
+
44
+ The vulnerability management team reserves the right to adjust the disclosure approach on a case-by-case basis, taking into account factors such as active exploitation, unusual attack surface, or coordination requirements with downstream vendors.
45
+
46
+ ## Prenotification policy
47
+
48
+ For certain security issues of CRITICAL, HIGH, or MODERATE severity level, we may prenotify certain organizations or vendors that ship vLLM. The purpose of this prenotification is to allow for a coordinated release of fixes for severe issues.
49
+
50
+ * This prenotification will be in the form of a private email notification. It may also include adding security contacts to the GitHub security advisory, typically a few days before release.
51
+
52
+ * If you wish to be added to the prenotification group, please send an email copying all the members of the [vulnerability management team](https://docs.vllm.ai/en/latest/contributing/vulnerability_management.html). Each vendor contact will be analyzed on a case-by-case basis.
53
+
54
+ * Organizations and vendors who either ship or use vLLM, are eligible to join the prenotification group if they meet at least one of the following qualifications
55
+ * Substantial internal deployment leveraging the upstream vLLM project.
56
+ * Established internal security teams and comprehensive compliance measures.
57
+ * Active and consistent contributions to the upstream vLLM project.
58
+
59
+ * We may withdraw organizations from receiving future prenotifications if they release fixes or any other information about issues before they are public. Group membership may also change based on policy refinements for who may be included.
asian_ai/performance/vLLM/rust-toolchain.toml ADDED
@@ -0,0 +1,2 @@
 
 
 
1
+ [toolchain]
2
+ channel = "1.95"
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_hermes.jinja ADDED
@@ -0,0 +1,130 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- macro json_to_python_type(json_spec) %}
2
+ {%- set basic_type_map = {
3
+ "string": "str",
4
+ "number": "float",
5
+ "integer": "int",
6
+ "boolean": "bool"
7
+ } %}
8
+
9
+ {%- if basic_type_map[json_spec.type] is defined %}
10
+ {{- basic_type_map[json_spec.type] }}
11
+ {%- elif json_spec.type == "array" %}
12
+ {{- "list[" + json_to_python_type(json_spec|items) + "]" }}
13
+ {%- elif json_spec.type == "object" %}
14
+ {%- if json_spec.additionalProperties is defined %}
15
+ {{- "dict[str, " + json_to_python_type(json_spec.additionalProperties) + ']' }}
16
+ {%- else %}
17
+ {{- "dict" }}
18
+ {%- endif %}
19
+ {%- elif json_spec.type is iterable %}
20
+ {{- "Union[" }}
21
+ {%- for t in json_spec.type %}
22
+ {{- json_to_python_type({"type": t}) }}
23
+ {%- if not loop.last %}
24
+ {{- "," }}
25
+ {%- endif %}
26
+ {%- endfor %}
27
+ {{- "]" }}
28
+ {%- else %}
29
+ {{- "Any" }}
30
+ {%- endif %}
31
+ {%- endmacro %}
32
+
33
+
34
+ {{- bos_token }}
35
+ {{- "<|im_start|>system\nYou are a function calling AI model. You are provided with function signatures within <tools></tools> XML tags. You may call one or more functions to assist with the user query. Don't make assumptions about what values to plug into functions. Here are the available tools: <tools> " }}
36
+ {%- if tools is iterable and tools | length > 0 %}
37
+ {%- for tool in tools %}
38
+ {%- if tool.function is defined %}
39
+ {%- set tool = tool.function %}
40
+ {%- endif %}
41
+ {{- '{"type": "function", "function": ' }}
42
+ {{- '{"name": "' + tool.name + '", ' }}
43
+ {{- '"description": "' + tool.name + '(' }}
44
+ {%- for param_name, param_fields in tool.parameters.properties|items %}
45
+ {{- param_name + ": " + json_to_python_type(param_fields) }}
46
+ {%- if not loop.last %}
47
+ {{- ", " }}
48
+ {%- endif %}
49
+ {%- endfor %}
50
+ {{- ")" }}
51
+ {%- if tool.return is defined %}
52
+ {{- " -> " + json_to_python_type(tool.return) }}
53
+ {%- endif %}
54
+ {{- " - " + tool.description + "\n\n" }}
55
+ {%- for param_name, param_fields in tool.parameters.properties|items %}
56
+ {%- if loop.first %}
57
+ {{- " Args:\n" }}
58
+ {%- endif %}
59
+ {{- " " + param_name + "(" + json_to_python_type(param_fields) + "): " + param_fields.description|trim }}
60
+ {%- endfor %}
61
+ {%- if tool.return is defined and tool.return.description is defined %}
62
+ {{- "\n Returns:\n " + tool.return.description }}
63
+ {%- endif %}
64
+ {{- '"' }}
65
+ {{- ', "parameters": ' }}
66
+ {%- if tool.parameters.properties | length == 0 %}
67
+ {{- "{}" }}
68
+ {%- else %}
69
+ {{- tool.parameters|tojson }}
70
+ {%- endif %}
71
+ {{- "}" }}
72
+ {%- if not loop.last %}
73
+ {{- "\n" }}
74
+ {%- endif %}
75
+ {%- endfor %}
76
+ {%- endif %}
77
+ {{- " </tools>" }}
78
+ {{- 'Use the following pydantic model json schema for each tool call you will make: {"properties": {"name": {"title": "Name", "type": "string"}, "arguments": {"title": "Arguments", "type": "object"}}, "required": ["name", "arguments"], "title": "FunctionCall", "type": "object"}}
79
+ ' }}
80
+ {{- "For each function call return a json object with function name and arguments within <tool_call></tool_call> XML tags as follows:
81
+ " }}
82
+ {{- "<tool_call>
83
+ " }}
84
+ {{- '{"name": <function-name>, "arguments": <args-dict>}
85
+ ' }}
86
+ {{- '</tool_call><|im_end|>' }}
87
+ {%- for message in messages %}
88
+ {%- if message.role == "user" or message.role == "system" or (message.role == "assistant" and message.tool_calls is not defined) %}
89
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
90
+ {%- elif message.role == "assistant" and message.tool_calls is defined %}
91
+ {{- '<|im_start|>' + message.role }}
92
+ {%- for tool_call in message.tool_calls %}
93
+ {{- '\n<tool_call>\n' }}
94
+ {%- if tool_call.function is defined %}
95
+ {%- set tool_call = tool_call.function %}
96
+ {%- endif %}
97
+ {{- '{' }}
98
+ {{- '"name": "' }}
99
+ {{- tool_call.name }}
100
+ {{- '"' }}
101
+ {%- if tool_call.arguments is defined %}
102
+ {{- ', ' }}
103
+ {{- '"arguments": ' }}
104
+ {{- tool_call.arguments|tojson }}
105
+ {%- endif %}
106
+ {{- '}' }}
107
+ {{- '\n</tool_call>' }}
108
+ {%- endfor %}
109
+ {{- '<|im_end|>\n' }}
110
+ {%- elif message.role == "tool" %}
111
+ {%- if loop.previtem and loop.previtem.role != "tool" %}
112
+ {{- '<|im_start|>tool\n' }}
113
+ {%- endif %}
114
+ {{- '<tool_response>\n' }}
115
+ {{- message.content }}
116
+ {%- if not loop.last %}
117
+ {{- '\n</tool_response>\n' }}
118
+ {%- else %}
119
+ {{- '\n</tool_response>' }}
120
+ {%- endif %}
121
+ {%- if not loop.last and loop.nextitem.role != "tool" %}
122
+ {{- '<|im_end|>' }}
123
+ {%- elif loop.last %}
124
+ {{- '<|im_end|>' }}
125
+ {%- endif %}
126
+ {%- endif %}
127
+ {%- endfor %}
128
+ {%- if add_generation_prompt %}
129
+ {{- '<|im_start|>assistant\n' }}
130
+ {%- endif %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_hunyuan_a13b.jinja ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% set loop_messages = messages %}
2
+ {% if tools %}
3
+ {% set weekday_map = {'Monday': '星期一', 'Tuesday': '星期二', 'Wednesday': '星期三', 'Thursday': '星期四', 'Friday': '星期五', 'Saturday': '星期六', 'Sunday': '星期日'} %}
4
+ {% set weekday_cn = weekday_map[strftime_now('%A')] %}
5
+ {% set datetime_str = strftime_now('%Y-%m-%d %H:%M:%S') %}
6
+ {% set datetime_str = datetime_str + ' ' + weekday_cn %}
7
+ {% for message in loop_messages %}
8
+ {% if 'content' in message %}
9
+ {% set content = message['content'] %}
10
+ {% else %}
11
+ {% set content = '' %}
12
+ {% endif %}
13
+ {% if loop.index0 == 0 %}
14
+ {% set content_tmp = '你是一位函数组合专家。你会得到一个问题和一组可能的函数。根据问题,你需要进行一个或多个函数/工具调用以实现目的。
15
+ 如果没有一个函数可以使用,请直接使用自然语言回复用户,以助手:开头。
16
+ 如果给定的问题缺少函数所需的参数,请使用自然语言进行提问,向用户询问必要信息,以助手:开头。
17
+ 如果调用结果已经足够回答用户问题,请对历史结果进行总结,使用自然语言回复用户,以助手:开头。
18
+ 你应该只在工具调用部分返回函数调用。如果你决定调用任何函数,你必须将其格式化为<tool_calls>[{"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},...]</tool_calls>。你不应该在回复中包含任何其他文本。以下是你可以调用的函数列表,格式为JSON。
19
+ ' %}
20
+ {% set content_tmp = content_tmp + '
21
+ ' + tools | tojson + '
22
+ ' %}
23
+ {% if message['role'] == 'system' %}
24
+ {% set content_tmp = content_tmp + '
25
+ 额外要求:
26
+ ' + content + '
27
+
28
+ 如果你决定返回函数调用,请将其格式化为<tool_calls>[{"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},...]</tool_calls>,不得包含其他文本。如果额外要求里有格式要求,请忽略,以此处为准。
29
+ 否则,请参考开头说的三种情况,以助手:开头进行回复。
30
+
31
+ 如果额外要求里有时间信息,就以额外要求里的时间为准,否则,参考当前时间:' + datetime_str %}
32
+ {% set content = '<|startoftext|>' + content_tmp + '<|extra_4|>' %}
33
+ {% elif message['role'] == 'user' %}
34
+ {% set content_tmp = content_tmp + '
35
+ 如果你决定返回函数调用,请将其格式化为<tool_calls>[{"name": "func_name1", "arguments": {"argument1": "value1", "argument2": "value2"}},...]</tool_calls>,不得包含其他文本。
36
+ 否则,请参考开头说的三种情况,以助手:开头进行回复。
37
+
38
+ 当前时间:' + datetime_str %}
39
+ {% set content_tmp = '<|startoftext|>' + content_tmp + '<|extra_4|>'%}
40
+ {% set content = content_tmp + '用户:' + content + '<|extra_0|>' %}
41
+ {% endif %}
42
+ {% else %}
43
+ {% if message['role'] == 'user' %}
44
+ {% set content = '用户:' + content + '<|extra_0|>' %}
45
+ {% elif message['role'] == 'assistant' %}
46
+ {% if 'tool_calls' in message %}
47
+ {% set tool_calls = message['tool_calls'] %}
48
+ {% set ns = namespace(tool_calls="[") %}
49
+ {% for tool_call in tool_calls %}
50
+ {% set function = tool_call['function'] %}
51
+ {% set name = function['name'] %}
52
+ {% set ns.tool_calls = ns.tool_calls + '{"name": "' + name + '", '%}
53
+ {% set arguments = function['arguments'] %}
54
+ {% if arguments is not string %}
55
+ {% set arguments = arguments | tojson %}
56
+ {% endif %}
57
+ {% set ns.tool_calls = ns.tool_calls + '"arguments": ' + arguments + '}' %}
58
+ {% if not loop.last %}
59
+ {% set ns.tool_calls = ns.tool_calls + ', '%}
60
+ {% endif %}
61
+ {% endfor %}
62
+ {% set ns.tool_calls = ns.tool_calls + ']' %}
63
+ {% set content = content + '<tool_calls>' + ns.tool_calls + '</tool_calls>' %}
64
+ {% else %}
65
+ {% set content = '助手:' + content %}
66
+ {% endif %}
67
+ {% set content = content + '<|eos|>' %}
68
+ {% elif message['role'] == 'tool' %}
69
+ {% if content is not string %}
70
+ {set content = content | tojson }
71
+ {% endif %}
72
+ {% set content = '<tool_response>' + content + '</tool_response>' %}
73
+ {% set content = content + '<|extra_0|>' %}
74
+ {% endif %}
75
+ {% endif %}
76
+ {{- content -}}
77
+ {% endfor %}
78
+ {% else %}
79
+ {% set context = {'has_head': true} %}
80
+ {% for message in loop_messages %}
81
+ {% if 'content' in message %}
82
+ {% set content = message['content'] %}
83
+ {% else %}
84
+ {% set content = '' %}
85
+ {% endif %}
86
+ {% if loop.index0 == 0 %}
87
+ {% if content == '' %}
88
+ {% set _ = context.update({'has_head': false}) %}
89
+ {% elif message['role'] == 'system' %}
90
+ {% set content = '<|startoftext|>' + content + '<|extra_4|>' %}
91
+ {% endif %}
92
+ {% endif %}
93
+ {% if message['role'] == 'user' %}
94
+ {% if loop.index0 == 1 and not context.has_head %}
95
+ {% set content = '<|startoftext|>' + content %}
96
+ {% endif %}
97
+ {% if loop.index0 == 1 and context.has_head %}
98
+ {% set content = content + '<|extra_0|>' %}
99
+ {% else %}
100
+ {% set content = '<|startoftext|>' + content + '<|extra_0|>' %}
101
+ {% endif %}
102
+ {% elif message['role'] == 'assistant' %}
103
+ {% set content = content + '<|eos|>' %}
104
+ {% elif message['role'] == 'tool' %}
105
+ {% set content = content + '<|extra_0|>' %}
106
+ {% endif %}
107
+ {{- content -}}
108
+ {% endfor %}
109
+ {% endif %}
110
+ {%- if enable_thinking is defined and enable_thinking is false %}
111
+ {{- '<think>\n\n</think>\n' }}
112
+ {%- endif %}
113
+
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_internlm2_tool.jinja ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if messages[0]["role"] == "system" %}
2
+ {%- set system_message = messages[0]["content"] %}
3
+ {%- set loop_messages = messages[1:] %}
4
+ {%- else %}
5
+ {%- set loop_messages = messages %}
6
+ {%- endif %}
7
+
8
+ {%- if not tools is defined %}
9
+ {%- set tools = none %}
10
+ {%- endif %}
11
+
12
+ {{- bos_token }}
13
+ {%- if system_message is defined %}
14
+ {{- "<|im_start|>system\n" + system_message + "<|im_end|>\n" }}
15
+ {%- endif %}
16
+
17
+ {%- if tools is not none %}
18
+ {{- "<|im_start|>system name=<|plugin|>\n[" }}
19
+ {%- for tool in tools %}
20
+ {{- tool.function|tojson }}
21
+ {%- if not loop.last %}
22
+ {{- ", " }}
23
+ {%- else %}
24
+ {{- "]" }}
25
+ {%- endif %}
26
+ {%- endfor %}
27
+ {{- "<|im_end|>\n" }}
28
+ {%- endif %}
29
+
30
+ {%- for message in loop_messages %}
31
+ {%- if message["role"] == "user" %}
32
+ {{- "<|im_start|>user\n" + message["content"] + "<|im_end|>\n"}}
33
+ {%- elif message.tool_calls is defined and message.tool_calls is not none %}
34
+ {%- set content = message["content"] if message["content"] else "" %}
35
+ {{- "<|im_start|>assistant\n" + content }}
36
+ {%- for tool_call in message.tool_calls %}
37
+ {%- set function=tool_call.function %}
38
+ {{- "<|action_start|><|plugin|>\n" }}
39
+ {{- '{"name": "' + function.name + '", '}}
40
+ {{- '"arguments": ' + function.arguments|tojson + '}' }}
41
+ {{- "<|action_end|>" }}
42
+ {%- endfor %}
43
+ {{- "<|im_end|>\n" }}
44
+ {%- elif message["role"] == "assistant" %}
45
+ {{- "<|im_start|>assistant\n" + message["content"] + "<|im_end|>\n"}}
46
+ {%- elif message["role"] == "tool_results" or message["role"] == "tool" or message["role"] == "function" %}
47
+ {%- if message.content is defined and message.content.content is defined %}
48
+ {%- set content = message.content.content %}
49
+ {%- else %}
50
+ {%- set content = message.content %}
51
+ {%- endif %}
52
+ {{- "<|im_start|>environment name=<|plugin|>\n" + content|string + "<|im_end|>\n" }}
53
+ {%- else %}
54
+ {{- raise_exception("Only user and assistant and tool_results and tool and function roles are supported, with the exception of an initial optional system message!") }}
55
+ {%- endif %}
56
+ {%- endfor %}
57
+
58
+ {%- if add_generation_prompt %}
59
+ {{- '<|im_start|>assistant\n' }}
60
+ {%- endif %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_llama3.1_json.jinja ADDED
@@ -0,0 +1,120 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token }}
2
+ {%- if custom_tools is defined %}
3
+ {%- set tools = custom_tools %}
4
+ {%- endif %}
5
+ {%- if not tools_in_user_message is defined %}
6
+ {#- Llama 3.1 doesn't pass all tests if the tools are in the system prompt #}
7
+ {%- set tools_in_user_message = true %}
8
+ {%- endif %}
9
+ {%- if not date_string is defined %}
10
+ {%- if strftime_now is defined %}
11
+ {%- set date_string = strftime_now("%d %b %Y") %}
12
+ {%- else %}
13
+ {%- set date_string = "26 Jul 2024" %}
14
+ {%- endif %}
15
+ {%- endif %}
16
+ {%- if not tools is defined %}
17
+ {%- set tools = none %}
18
+ {%- endif %}
19
+
20
+ {#- This block extracts the system message, so we can slot it into the right place. #}
21
+ {%- if messages[0]['role'] == 'system' %}
22
+ {%- if messages[0]['content'] is string %}
23
+ {%- set system_message = messages[0]['content']|trim %}
24
+ {%- else %}
25
+ {%- set system_message = messages[0]['content'][0]['text']|trim %}
26
+ {%- endif %}
27
+ {%- set messages = messages[1:] %}
28
+ {%- else %}
29
+ {%- if tools is not none %}
30
+ {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
31
+ {%- else %}
32
+ {%- set system_message = "" %}
33
+ {%- endif %}
34
+ {%- endif %}
35
+
36
+ {#- System message #}
37
+ {{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
38
+ {%- if tools is not none %}
39
+ {{- "Environment: ipython\n" }}
40
+ {%- endif %}
41
+ {{- "Cutting Knowledge Date: December 2023\n" }}
42
+ {{- "Today Date: " + date_string + "\n\n" }}
43
+ {%- if tools is not none and not tools_in_user_message %}
44
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call. " }}
45
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
46
+ {{- "Do not use variables.\n\n" }}
47
+ {%- for t in tools %}
48
+ {{- t | tojson(indent=4) }}
49
+ {{- "\n\n" }}
50
+ {%- endfor %}
51
+ {%- endif %}
52
+ {{- system_message }}
53
+ {{- "<|eot_id|>" }}
54
+
55
+ {#- Custom tools are passed in a user message with some extra guidance #}
56
+ {%- if tools_in_user_message and not tools is none %}
57
+ {#- Extract the first user message so we can plug it in here #}
58
+ {%- if messages | length != 0 %}
59
+ {%- if messages[0]['content'] is string %}
60
+ {%- set first_user_message = messages[0]['content']|trim %}
61
+ {%- else %}
62
+ {%- set first_user_message = messages[0]['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | map('trim') | join('\n') %}
63
+ {%- endif %}
64
+ {%- set messages = messages[1:] %}
65
+ {%- else %}
66
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
67
+ {%- endif %}
68
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
69
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
70
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
71
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
72
+ {{- "Do not use variables.\n\n" }}
73
+ {%- for t in tools %}
74
+ {{- t | tojson(indent=4) }}
75
+ {{- "\n\n" }}
76
+ {%- endfor %}
77
+ {{- first_user_message + "<|eot_id|>"}}
78
+ {%- endif %}
79
+
80
+ {%- for message in messages %}
81
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
82
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }}
83
+ {%- if message['content'] is string %}
84
+ {{- message['content'] | trim}}
85
+ {%- else %}
86
+ {%- for content in message['content'] %}
87
+ {%- if content['type'] == 'text' %}
88
+ {{- content['text'] | trim }}
89
+ {%- endif %}
90
+ {%- endfor %}
91
+ {%- endif %}
92
+ {{- '<|eot_id|>' }}
93
+ {%- elif 'tool_calls' in message %}
94
+ {%- if not message.tool_calls|length == 1 %}
95
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
96
+ {%- endif %}
97
+ {%- set tool_call = message.tool_calls[0].function %}
98
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
99
+ {{- '{"name": "' + tool_call.name + '", ' }}
100
+ {{- '"parameters": ' }}
101
+ {{- tool_call.arguments | tojson }}
102
+ {{- "}" }}
103
+ {{- "<|eot_id|>" }}
104
+ {%- elif message.role == "tool" or message.role == "ipython" %}
105
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
106
+ {%- if message.content is string %}
107
+ {{- { "output": message.content } | tojson }}
108
+ {%- else %}
109
+ {%- for content in message['content'] %}
110
+ {%- if content['type'] == 'text' %}
111
+ {{- { "output": content['text'] } | tojson }}
112
+ {%- endif %}
113
+ {%- endfor %}
114
+ {%- endif %}
115
+ {{- "<|eot_id|>" }}
116
+ {%- endif %}
117
+ {%- endfor %}
118
+ {%- if add_generation_prompt %}
119
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
120
+ {%- endif %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_llama3.2_json.jinja ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token }}
2
+ {%- if custom_tools is defined %}
3
+ {%- set tools = custom_tools %}
4
+ {%- endif %}
5
+ {%- if not tools_in_user_message is defined %}
6
+ {%- set tools_in_user_message = false %}
7
+ {%- endif %}
8
+ {%- if not date_string is defined %}
9
+ {%- if strftime_now is defined %}
10
+ {%- set date_string = strftime_now("%d %b %Y") %}
11
+ {%- else %}
12
+ {%- set date_string = "26 Jul 2024" %}
13
+ {%- endif %}
14
+ {%- endif %}
15
+ {%- if not tools is defined %}
16
+ {%- set tools = none %}
17
+ {%- endif %}
18
+
19
+ {#- Find out if there are any images #}
20
+ {% set image_ns = namespace(has_images=false) %}
21
+ {%- for message in messages %}
22
+ {%- for content in message['content'] %}
23
+ {%- if content['type'] == 'image' %}
24
+ {%- set image_ns.has_images = true %}
25
+ {%- endif %}
26
+ {%- endfor %}
27
+ {%- endfor %}
28
+
29
+ {#- This block extracts the system message, so we can slot it into the right place. #}
30
+ {%- if messages[0]['role'] == 'system' %}
31
+ {%- if messages[0]['content'] is string %}
32
+ {%- set system_message = messages[0]['content']|trim %}
33
+ {%- else %}
34
+ {%- set system_message = messages[0]['content'][0]['text']|trim %}
35
+ {%- endif %}
36
+ {%- set messages = messages[1:] %}
37
+ {%- else %}
38
+ {%- if tools is not none %}
39
+ {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
40
+ {%- else %}
41
+ {%- set system_message = "" %}
42
+ {%- endif %}
43
+ {%- endif %}
44
+
45
+ {#- System message if there are no images, if the user supplied one, or if tools are used (default tool system message) #}
46
+ {%- if system_message or not image_ns.has_images %}
47
+ {{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
48
+ {%- if tools is not none %}
49
+ {{- "Environment: ipython\n" }}
50
+ {%- endif %}
51
+ {{- "Cutting Knowledge Date: December 2023\n" }}
52
+ {{- "Today Date: " + date_string + "\n\n" }}
53
+ {%- if tools is not none and not tools_in_user_message %}
54
+ {{- "You have access to the following functions. To call a function, please respond with JSON for a function call. " }}
55
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
56
+ {{- "Do not use variables.\n\n" }}
57
+ {%- for t in tools %}
58
+ {{- t | tojson(indent=4) }}
59
+ {{- "\n\n" }}
60
+ {%- endfor %}
61
+ {%- endif %}
62
+ {{- system_message }}
63
+ {{- "<|eot_id|>" }}
64
+ {%- endif %}
65
+
66
+ {#- Custom tools are passed in a user message with some extra guidance #}
67
+ {%- if tools_in_user_message and not tools is none %}
68
+ {#- Extract the first user message so we can plug it in here #}
69
+ {%- if messages | length != 0 %}
70
+ {%- if messages[0]['content'] is string %}
71
+ {%- set first_user_message = messages[0]['content']|trim %}
72
+ {%- else %}
73
+ {%- set first_user_message = messages[0]['content'] | selectattr('type', 'equalto', 'text') | map(attribute='text') | map('trim') | join('\n') %}
74
+ {%- endif %}
75
+ {%- set messages = messages[1:] %}
76
+ {%- else %}
77
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
78
+ {%- endif %}
79
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
80
+ {{- "Given the following functions, please respond with a JSON for a function call " }}
81
+ {{- "with its proper arguments that best answers the given prompt.\n\n" }}
82
+ {{- 'Respond in the format {"name": function name, "parameters": dictionary of argument name and its value}. ' }}
83
+ {{- "Do not use variables.\n\n" }}
84
+ {%- for t in tools %}
85
+ {{- t | tojson(indent=4) }}
86
+ {{- "\n\n" }}
87
+ {%- endfor %}
88
+ {{- first_user_message + "<|eot_id|>"}}
89
+ {%- endif %}
90
+
91
+ {%- for message in messages %}
92
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
93
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n' }}
94
+ {%- if message['content'] is string %}
95
+ {{- message['content'] | trim}}
96
+ {%- else %}
97
+ {%- for content in message['content'] %}
98
+ {%- if content['type'] == 'image' %}
99
+ {{- '<|image|>' }}
100
+ {%- elif content['type'] == 'text' %}
101
+ {{- content['text'] | trim }}
102
+ {%- endif %}
103
+ {%- endfor %}
104
+ {%- endif %}
105
+ {{- '<|eot_id|>' }}
106
+ {%- elif 'tool_calls' in message %}
107
+ {%- if not message.tool_calls|length == 1 %}
108
+ {{- raise_exception("This model only supports single tool-calls at once!") }}
109
+ {%- endif %}
110
+ {%- set tool_call = message.tool_calls[0].function %}
111
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
112
+ {{- '{"name": "' + tool_call.name + '", ' }}
113
+ {{- '"parameters": ' }}
114
+ {{- tool_call.arguments | tojson }}
115
+ {{- "}" }}
116
+ {{- "<|eot_id|>" }}
117
+ {%- elif message.role == "tool" or message.role == "ipython" %}
118
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
119
+ {%- if message.content is string %}
120
+ {{- { "output": message.content } | tojson }}
121
+ {%- else %}
122
+ {%- for content in message['content'] %}
123
+ {%- if content['type'] == 'text' %}
124
+ {{- { "output": content['text'] } | tojson }}
125
+ {%- endif %}
126
+ {%- endfor %}
127
+ {%- endif %}
128
+ {{- "<|eot_id|>" }}
129
+ {%- endif %}
130
+ {%- endfor %}
131
+ {%- if add_generation_prompt %}
132
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
133
+ {%- endif %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_llama3.2_pythonic.jinja ADDED
@@ -0,0 +1,98 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token }}
2
+ {%- if custom_tools is defined %}
3
+ {%- set tools = custom_tools %}
4
+ {%- endif %}
5
+ {%- if not tools_in_user_message is defined %}
6
+ {%- set tools_in_user_message = false %}
7
+ {%- endif %}
8
+ {%- if not date_string is defined %}
9
+ {%- if strftime_now is defined %}
10
+ {%- set date_string = strftime_now("%d %b %Y") %}
11
+ {%- else %}
12
+ {%- set date_string = "26 Jul 2024" %}
13
+ {%- endif %}
14
+ {%- endif %}
15
+ {%- if not tools is defined %}
16
+ {%- set tools = none %}
17
+ {%- endif %}
18
+
19
+ {#- This block extracts the system message, so we can slot it into the right place. #}
20
+ {%- if messages[0]['role'] == 'system' %}
21
+ {%- set system_message = messages[0]['content']|trim %}
22
+ {%- set messages = messages[1:] %}
23
+ {%- else %}
24
+ {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language. When you receive a tool call response, use the output to format an answer to the original user question." %}
25
+ {%- endif %}
26
+
27
+ {#- System message #}
28
+ {{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
29
+ {%- if tools is not none %}
30
+ {{- "Environment: ipython\n" }}
31
+ {%- endif %}
32
+ {{- "Cutting Knowledge Date: December 2023\n" }}
33
+ {{- "Today Date: " + date_string + "\n\n" }}
34
+ {%- if tools is not none and not tools_in_user_message %}
35
+ {{- "You have access to the following functions. To call functions, please respond with a python list of the calls. " }}
36
+ {{- 'Respond in the format [func_name1(params_name1=params_value1, params_name2=params_value2...), func_name2(params)] ' }}
37
+ {{- "Do not use variables.\n\n" }}
38
+ {%- for t in tools %}
39
+ {{- t | tojson(indent=4) }}
40
+ {{- "\n\n" }}
41
+ {%- endfor %}
42
+ {%- endif %}
43
+ {{- system_message }}
44
+ {{- "<|eot_id|>" }}
45
+
46
+ {#- Custom tools are passed in a user message with some extra guidance #}
47
+ {%- if tools_in_user_message and not tools is none %}
48
+ {#- Extract the first user message so we can plug it in here #}
49
+ {%- if messages | length != 0 %}
50
+ {%- set first_user_message = messages[0]['content']|trim %}
51
+ {%- set messages = messages[1:] %}
52
+ {%- else %}
53
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
54
+ {%- endif %}
55
+ {{- '<|start_header_id|>user<|end_header_id|>\n\n' -}}
56
+ {{- "Given the following functions, please respond with a python list for function calls " }}
57
+ {{- "with their proper arguments to best answer the given prompt.\n\n" }}
58
+ {{- 'Respond in the format [func_name1(params_name1=params_value1, params_name2=params_value2...), func_name2(params)] ' }}
59
+ {{- "Do not use variables.\n\n" }}
60
+ {%- for t in tools %}
61
+ {{- t | tojson(indent=4) }}
62
+ {{- "\n\n" }}
63
+ {%- endfor %}
64
+ {{- first_user_message + "<|eot_id|>"}}
65
+ {%- endif %}
66
+
67
+ {%- for message in messages %}
68
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
69
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
70
+ {%- elif 'tool_calls' in message %}
71
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n[' -}}
72
+ {%- for tool_call in message.tool_calls %}
73
+ {%- if tool_call.function is defined %}
74
+ {%- set tool_call = tool_call.function %}
75
+ {%- endif %}
76
+ {{- tool_call.name + '(' -}}
77
+ {%- for param in tool_call.arguments %}
78
+ {{- param + '=' -}}
79
+ {{- "%s" | format(tool_call.arguments[param]) -}}
80
+ {% if not loop.last %}, {% endif %}
81
+ {%- endfor %}
82
+ {{- ')' -}}
83
+ {% if not loop.last %}, {% endif %}
84
+ {%- endfor %}
85
+ {{- ']<|eot_id|>' -}}
86
+ {%- elif message.role == "tool" or message.role == "ipython" %}
87
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
88
+ {%- if message.content is mapping %}
89
+ {{- message.content | tojson }}
90
+ {%- else %}
91
+ {{- { "output": message.content } | tojson }}
92
+ {%- endif %}
93
+ {{- "<|eot_id|>" }}
94
+ {%- endif %}
95
+ {%- endfor %}
96
+ {%- if add_generation_prompt %}
97
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
98
+ {%- endif %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_llama4_json.jinja ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- macro is_array_of_type_objects(var) -%}
2
+ {%- if var is iterable and var is not string -%}
3
+ {%- set valid = true -%}
4
+ {%- for item in var -%}
5
+ {%- if 'type' not in item -%}
6
+ {%- set valid = false -%}
7
+ {%- break -%}
8
+ {%- endif -%}
9
+ {%- endfor -%}
10
+ {{ valid }}
11
+ {%- else -%}
12
+ {{ false }}
13
+ {%- endif -%}
14
+ {%- endmacro %}
15
+
16
+ {%- macro render_message(message) %}
17
+ {%- if message['content'] is string %}
18
+ {{- message['content']|trim }}
19
+ {%- elif is_array_of_type_objects(data) == 'True' %}
20
+ {%- for content in message['content'] %}
21
+ {%- if content['type'] == 'image' %}
22
+ {{- '<|image|>' }}
23
+ {%- elif content['type'] == 'text' %}
24
+ {{- content['text']|trim }}
25
+ {%- endif %}
26
+ {%- endfor %}
27
+ {%- else %}
28
+ {{- message['content']|tojson }}
29
+ {%- endif %}
30
+ {%- endmacro %}
31
+
32
+ {{- bos_token }}
33
+ {%- if custom_tools is defined %}
34
+ {%- set tools = custom_tools %}
35
+ {%- endif %}
36
+ {%- if not tools_in_user_message is defined %}
37
+ {%- set tools_in_user_message = true %}
38
+ {%- endif %}
39
+ {%- if not tools is defined %}
40
+ {%- set tools = none %}
41
+ {%- endif %}
42
+
43
+ {#- This block extracts the system message, so we can slot it into the right place. #}
44
+ {%- if messages[0]['role'] == 'system' %}
45
+ {%- set system_message = messages[0] %}
46
+ {%- set messages = messages[1:] %}
47
+ {%- else %}
48
+ {%- set system_message = ({ "content": "You are a helpful assistant with tool calling "
49
+ "capabilities. Only reply with a tool call if the function exists in the "
50
+ "library provided by the user. If it doesn't exist, just reply directly in "
51
+ "natural language. When you receive a tool call response, use the output to "
52
+ "format an answer to the original user question."}) %}
53
+ {%- endif %}
54
+
55
+ {%- set tool_lib_preamble = 'Tools: You have access to the following tools. You might need to use one '
56
+ 'or more function/tool calls to fulfill the task. \n'
57
+ 'If none are needed, then proceed to the response.\n\n'
58
+ 'Tool Call Syntax: You can call tools using the following syntax:\n'
59
+ '{"name": function name, "parameters": dictionary of argument name and its value}.\n'
60
+ 'Separate multiple function calls by "; ". Do not use variables.\n'
61
+ 'Do not include anything else when calling the tools with the syntax above.\n\n'
62
+ 'Here is a list of functions in JSON format that you can invoke.\n' %}
63
+
64
+ {{- "<|header_start|>system<|header_end|>\n\n" }}
65
+ {%- if tools is not none and not tools_in_user_message %}
66
+ {{- tool_lib_preamble }}
67
+ {%- for t in tools %}
68
+ {{- t | tojson(indent=4) }}
69
+ {{- "\n\n" }}
70
+ {%- endfor %}
71
+ {%- endif %}
72
+ {{- render_message(system_message) }}
73
+ {{ "<|eot|>\n" }}
74
+
75
+ {#- Custom tools are passed in a user message with some extra guidance #}
76
+ {%- if tools_in_user_message and not tools is none %}
77
+ {#- Extract the first user message so we can plug it in here #}
78
+ {%- if messages | length != 0 %}
79
+ {%- set first_user_message = messages[0] %}
80
+ {%- set messages = messages[1:] %}
81
+ {%- else %}
82
+ {{- raise_exception("Cannot put tools in the first user message when there's no first user message!") }}
83
+ {%- endif %}
84
+ {{- '<|header_start|>user<|header_end|>\n\n' }}
85
+ {{- tool_lib_preamble }}
86
+ {%- for t in tools %}
87
+ {{- t | tojson(indent=4) }}
88
+ {{- "\n\n" }}
89
+ {%- endfor %}
90
+ {{- render_message(first_user_message) + "\n<|eot|>"}}
91
+ {%- endif %}
92
+
93
+ {%- for message in messages %}
94
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
95
+ {{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }}
96
+ {{- render_message(message) }}
97
+ {{- "\n<|eot|>" }}
98
+ {%- elif 'tool_calls' in message and message.tool_calls|length > 0 %}
99
+ {{- '\n<|header_start|>assistant<|header_end|>\n\n' -}}
100
+ {{- render_message(message) }}
101
+ {%- for tool_call in message.tool_calls %}
102
+ {{- '{"name": "' + tool_call.function.name + '", ' }}
103
+ {{- '"parameters": ' }}
104
+ {{- tool_call.function.arguments | tojson }}
105
+ {{- "}" }}
106
+ {%- endfor %}
107
+ {{- "\n<|eot|>" }}
108
+ {%- elif message.role == "tool" or message.role == "ipython" %}
109
+ {{- "\n<|header_start|>ipython<|header_end|>\n\n" }}
110
+ {{- render_message(message) }}
111
+ {{- "\n<|eom|>" }}
112
+ {%- endif %}
113
+ {%- endfor %}
114
+ {%- if add_generation_prompt %}
115
+ {{- '\n<|header_start|>assistant<|header_end|>\n\n' }}
116
+ {%- endif %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_llama4_pythonic.jinja ADDED
@@ -0,0 +1,111 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token }}
2
+ {%- if custom_tools is defined and custom_tools%}
3
+ {%- set tools = custom_tools %}
4
+ {%- endif %}
5
+ {%- if tools is defined and tools %}
6
+ {%- set tool_definition = tool_definition ~ (tools | tojson(indent=4)) %}
7
+ {%- else %}
8
+ {%- set tools = none %}
9
+ {%- endif %}
10
+
11
+
12
+ {#- This block extracts the system message, so we can slot it into the right place. #}
13
+ {%- if messages[0]['role'] == 'system' %}
14
+ {%- set user_provided_system_message = true %}
15
+ {%- if messages[0]['content'] is string %}
16
+ {%- set system_message = messages[0]['content']|trim %}
17
+ {%- else %}
18
+ {%- set system_message = messages[0]['content'][0]['text']|trim %}
19
+ {%- endif %}
20
+ {%- set messages = messages[1:] %}
21
+ {%- else %}
22
+ {%- if tools is not none %}
23
+ {#- Since not system_message was provided by user, if tool is provided, system_message is now default tool system message #}
24
+ {#- This system message is from llama website:https://www.llama.com/docs/model-cards-and-prompt-formats/llama4/ #}
25
+ {%- set system_message = "You are a helpful assistant and an expert in function composition. You can answer general questions using your internal knowledge OR invoke functions when necessary. Follow these strict guidelines:\n\n1. FUNCTION CALLS:\n- ONLY use functions that are EXPLICITLY listed in the function list below\n- If NO functions are listed (empty function list []), respond ONLY with internal knowledge or \"I don't have access to [Unavailable service] information\"\n- If a function is not in the list, respond ONLY with internal knowledge or \"I don't have access to [Unavailable service] information\"\n- If ALL required parameters are present AND the query EXACTLY matches a listed function's purpose: output ONLY the function call(s)\n- Use exact format: [func_name1(param1=value1, param2=value2), func_name2(...)]\nExamples:\nCORRECT: [get_weather(location=\"Vancouver\"), calculate_route(start=\"Boston\", end=\"New York\")] <- Only if get_weather and calculate_route are in function list\nINCORRECT: get_weather(location=\"New York\")\nINCORRECT: Let me check the weather: [get_weather(location=\"New York\")]\nINCORRECT: [get_events(location=\"Singapore\")] <- If function not in list\n\n2. RESPONSE RULES:\n- For pure function requests matching a listed function: ONLY output the function call(s)\n- For knowledge questions: ONLY output text\n- For missing parameters: ONLY request the specific missing parameters\n- For unavailable services (not in function list): output ONLY with internal knowledge or \"I don't have access to [Unavailable service] information\". Do NOT execute a function call.\n- If the query asks for information beyond what a listed function provides: output ONLY with internal knowledge about your limitations\n- NEVER combine text and function calls in the same response\n- NEVER suggest alternative functions when the requested service is unavailable\n- NEVER create or invent new functions not listed below\n\n3. STRICT BOUNDARIES:\n- ONLY use functions from the list below - no exceptions\n- NEVER use a function as an alternative to unavailable information\n- NEVER call functions not present in the function list\n- NEVER add explanatory text to function calls\n- NEVER respond with empty brackets\n- Use proper Python/JSON syntax for function calls\n- Check the function list carefully before responding\n\n4. TOOL RESPONSE HANDLING:\n- When receiving tool responses: provide concise, natural language responses\n- Don't repeat tool response verbatim\n- Don't add supplementary information\n\nHere is a list of functions in JSON format that you can invoke:\n" %}
26
+ {%- else %}
27
+ {%- set system_message = "" %}
28
+ {%- endif %}
29
+ {%- endif %}
30
+ {#- Now writing the system message: use the user provided system message if user_provided_system_message, else default tool system message if tools presented #}
31
+ {%- if system_message %}
32
+ {#- always use user provided system message to override default tool system message #}
33
+ {{- "<|header_start|>system<|header_end|>\n\n" }}
34
+ {{- system_message }}
35
+ {%- if user_provided_system_message and tools %}
36
+ {{- "\nHere is a list of functions in JSON format that you can invoke. Use exact format: [func_name1(param1=value1, param2=value2), func_name2(...)]\n" }}
37
+ {{- tool_definition -}}
38
+ {%- elif tool_definition %}
39
+ {{- tool_definition -}}
40
+ {%- endif %}
41
+ {{- "<|eot|>" }}
42
+ {%- endif %}
43
+
44
+ {#- Now deal with all other messages #}
45
+ {%- for message in messages %}
46
+ {#- Base case: messages that are not from tool role and has empty tool_call list #}
47
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or ('tool_calls' in message and message.tool_calls|length != 0 )) %}
48
+ {{- '<|header_start|>' + message['role'] + '<|header_end|>\n\n' }}
49
+ {%- if message['content'] is string %}
50
+ {{- message['content'] }}
51
+ {%- else %}
52
+ {%- for content in message['content'] %}
53
+ {%- if content['type'] == 'image' %}
54
+ {{- '<|image|>' }}
55
+ {%- elif content['type'] == 'text' %}
56
+ {{- content['text'] | trim }}
57
+ {%- endif %}
58
+ {%- endfor %}
59
+ {%- endif %}
60
+ {{- "<|eot|>" }}
61
+ {#- Tool case: messages has non-empty tool_call list, must from assistant #}
62
+ {%- elif 'tool_calls' in message %}
63
+ {#- assume tool_calls are always coming from assistant #}
64
+ {%- if message.role == 'assistant' %}
65
+ {{- '<|header_start|>assistant<|header_end|>\n\n' -}}
66
+ {%- if message['content'] is string %}
67
+ {{- message['content'] }}
68
+ {%- else %}
69
+ {%- for content in message['content'] %}
70
+ {%- if content['type'] == 'image' %}
71
+ {{- '<|image|>' }}
72
+ {%- elif content['type'] == 'text' %}
73
+ {{- content['text'] }}
74
+ {%- endif %}
75
+ {%- endfor %}
76
+ {%- endif %}
77
+ {{- "[" }}
78
+ {%- for tool_call in message.tool_calls %}
79
+ {%- if tool_call.function is defined %}
80
+ {%- set tool_call = tool_call.function %}
81
+ {%- endif %}
82
+ {{- tool_call.name + '(' -}}
83
+ {%- for param in tool_call.arguments %}
84
+ {{- param + '="' -}}
85
+ {{- "%s" | format(tool_call.arguments[param]) -}}
86
+ {{- '"' -}}
87
+ {% if not loop.last %}, {% endif %}
88
+ {%- endfor %}
89
+ {{- ')' -}}
90
+ {% if not loop.last %}, {% endif %}
91
+ {%- endfor %}
92
+ {{- "]<|eot|>" }}
93
+ {%- endif %}
94
+ {#- Tool_response case: messages are from tool_response #}
95
+ {%- elif message.role == "tool" or message.role == "ipython" %}
96
+ {{- "<|header_start|>ipython<|header_end|>\n\n" }}
97
+ {%- if message.content is string %}
98
+ {{- message.content | tojson }}
99
+ {%- else %}
100
+ {%- for content in message['content'] %}
101
+ {%- if content['type'] == 'text' %}
102
+ {{- content['text'] | tojson }}
103
+ {%- endif %}
104
+ {%- endfor %}
105
+ {%- endif %}
106
+ {{- "<|eot|>" }}
107
+ {%- endif %}
108
+ {%- endfor %}
109
+ {%- if add_generation_prompt %}
110
+ {{- '<|header_start|>assistant<|header_end|>\n\n' }}
111
+ {%- endif %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_mistral.jinja ADDED
@@ -0,0 +1,86 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if messages[0]["role"] == "system" %}
2
+ {%- set system_message = messages[0]["content"] %}
3
+ {%- set loop_messages = messages[1:] %}
4
+ {%- else %}
5
+ {%- set loop_messages = messages %}
6
+ {%- endif %}
7
+ {%- if not tools is defined %}
8
+ {%- set tools = none %}
9
+ {%- endif %}
10
+ {%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %}
11
+
12
+ {%- for message in loop_messages | rejectattr("role", "equalto", "tool") | rejectattr("role", "equalto", "tool_results") | selectattr("tool_calls", "undefined") %}
13
+ {%- if (message["role"] == "user") != (loop.index0 % 2 == 0) %}
14
+ {{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }}
15
+ {%- endif %}
16
+ {%- endfor %}
17
+
18
+ {{- bos_token }}
19
+ {%- for message in loop_messages %}
20
+ {%- if message["role"] == "user" %}
21
+ {%- if tools is not none and (message == user_messages[-1]) %}
22
+ {{- "[AVAILABLE_TOOLS] [" }}
23
+ {%- for tool in tools %}
24
+ {%- set tool = tool.function %}
25
+ {{- '{"type": "function", "function": {' }}
26
+ {%- for key, val in tool.items() if key != "return" %}
27
+ {%- if val is string %}
28
+ {{- '"' + key + '": "' + val + '"' }}
29
+ {%- else %}
30
+ {{- '"' + key + '": ' + val|tojson }}
31
+ {%- endif %}
32
+ {%- if not loop.last %}
33
+ {{- ", " }}
34
+ {%- endif %}
35
+ {%- endfor %}
36
+ {{- "}}" }}
37
+ {%- if not loop.last %}
38
+ {{- ", " }}
39
+ {%- else %}
40
+ {{- "]" }}
41
+ {%- endif %}
42
+ {%- endfor %}
43
+ {{- "[/AVAILABLE_TOOLS]" }}
44
+ {%- endif %}
45
+ {%- if loop.last and system_message is defined %}
46
+ {{- "[INST] " + system_message + "\n\n" + message["content"] + "[/INST]" }}
47
+ {%- else %}
48
+ {{- "[INST] " + message["content"] + "[/INST]" }}
49
+ {%- endif %}
50
+ {%- elif message["role"] == "tool_calls" or message.tool_calls is defined %}
51
+ {%- if message.tool_calls is defined %}
52
+ {%- set tool_calls = message.tool_calls %}
53
+ {%- else %}
54
+ {%- set tool_calls = message.content %}
55
+ {%- endif %}
56
+ {{- "[TOOL_CALLS] [" }}
57
+ {%- for tool_call in tool_calls %}
58
+ {%- set out = tool_call.function|tojson %}
59
+ {{- out[:-1] }}
60
+ {%- if not tool_call.id is defined or tool_call.id|length < 9 %}
61
+ {{- raise_exception("Tool call IDs should be alphanumeric strings with length >= 9! (1)" + tool_call.id) }}
62
+ {%- endif %}
63
+ {{- ', "id": "' + tool_call.id[-9:] + '"}' }}
64
+ {%- if not loop.last %}
65
+ {{- ", " }}
66
+ {%- else %}
67
+ {{- "]" + eos_token }}
68
+ {%- endif %}
69
+ {%- endfor %}
70
+ {%- elif message["role"] == "assistant" %}
71
+ {{- " " + message["content"] + eos_token }}
72
+ {%- elif message["role"] == "tool_results" or message["role"] == "tool" %}
73
+ {%- if message.content is defined and message.content.content is defined %}
74
+ {%- set content = message.content.content %}
75
+ {%- else %}
76
+ {%- set content = message.content %}
77
+ {%- endif %}
78
+ {{- '[TOOL_RESULTS] {"content": ' + content|string + ", " }}
79
+ {%- if not message.tool_call_id is defined or message.tool_call_id|length < 9 %}
80
+ {{- raise_exception("Tool call IDs should be alphanumeric strings with length >= 9! (2)" + message.tool_call_id) }}
81
+ {%- endif %}
82
+ {{- '"call_id": "' + message.tool_call_id[-9:] + '"}[/TOOL_RESULTS]' }}
83
+ {%- else %}
84
+ {{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}
85
+ {%- endif %}
86
+ {%- endfor %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_mistral3.jinja ADDED
@@ -0,0 +1,126 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- set today = strftime_now("%Y-%m-%d") %}
2
+ {%- set default_system_message = "You are Mistral Small 3, a Large Language Model (LLM) created by Mistral AI, a French startup headquartered in Paris.\nYour knowledge base was last updated on 2023-10-01. The current date is " + today + ".\n\nWhen you're not sure about some information, you say that you don't have the information and don't make up anything.\nIf the user's question is not clear, ambiguous, or does not provide enough context for you to accurately answer the question, you do not try to answer it right away and you rather ask the user to clarify their request (e.g. \"What are some good restaurants around me?\" => \"Where are you?\" or \"When is the next flight to Tokyo\" => \"Where do you travel from?\")" %}
3
+
4
+ {{- bos_token }}
5
+
6
+ {%- if messages[0]['role'] == 'system' %}
7
+ {%- if messages[0]['content'] is string %}
8
+ {%- set system_message = messages[0]['content'] %}
9
+ {%- set loop_messages = messages[1:] %}
10
+ {%- else %}
11
+ {%- set system_message = messages[0]['content'][0]['text'] %}
12
+ {%- set loop_messages = messages[1:] %}
13
+ {%- endif %}
14
+ {%- else %}
15
+ {%- set system_message = default_system_message %}
16
+ {%- set loop_messages = messages %}
17
+ {%- endif %}
18
+ {%- if not tools is defined %}
19
+ {%- set tools = none %}
20
+ {%- elif tools is not none %}
21
+ {%- set parallel_tool_prompt = "You are a helpful assistant that can call tools. If you call one or more tools, format them in a single JSON array or objects, where each object is a tool call, not as separate objects outside of an array or multiple arrays. Use the format [{\"name\": tool call name, \"arguments\": tool call arguments}, additional tool calls] if you call more than one tool. If you call tools, do not attempt to interpret them or otherwise provide a response until you receive a tool call result that you can interpret for the user." %}
22
+ {%- if system_message is defined %}
23
+ {%- set system_message = parallel_tool_prompt + "\n\n" + system_message %}
24
+ {%- else %}
25
+ {%- set system_message = parallel_tool_prompt %}
26
+ {%- endif %}
27
+ {%- endif %}
28
+ {{- '[SYSTEM_PROMPT]' + system_message + '[/SYSTEM_PROMPT]' }}
29
+
30
+ {%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %}
31
+
32
+ {%- set filtered_messages = [] %}
33
+ {%- for message in loop_messages %}
34
+ {%- if message["role"] not in ["tool", "tool_results"] and not message.get("tool_calls") %}
35
+ {%- set filtered_messages = filtered_messages + [message] %}
36
+ {%- endif %}
37
+ {%- endfor %}
38
+
39
+ {%- for message in filtered_messages %}
40
+ {%- if (message["role"] == "user") != (loop.index0 % 2 == 0) %}
41
+ {{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }}
42
+ {%- endif %}
43
+ {%- endfor %}
44
+
45
+ {%- for message in loop_messages %}
46
+ {%- if message["role"] == "user" %}
47
+ {%- if tools is not none and (message == user_messages[-1]) %}
48
+ {{- "[AVAILABLE_TOOLS] [" }}
49
+ {%- for tool in tools %}
50
+ {%- set tool = tool.function %}
51
+ {{- '{"type": "function", "function": {' }}
52
+ {%- for key, val in tool.items() if key != "return" %}
53
+ {%- if val is string %}
54
+ {{- '"' + key + '": "' + val + '"' }}
55
+ {%- else %}
56
+ {{- '"' + key + '": ' + val|tojson }}
57
+ {%- endif %}
58
+ {%- if not loop.last %}
59
+ {{- ", " }}
60
+ {%- endif %}
61
+ {%- endfor %}
62
+ {{- "}}" }}
63
+ {%- if not loop.last %}
64
+ {{- ", " }}
65
+ {%- else %}
66
+ {{- "]" }}
67
+ {%- endif %}
68
+ {%- endfor %}
69
+ {{- "[/AVAILABLE_TOOLS]" }}
70
+ {%- endif %}
71
+ {%- if message['content'] is string %}
72
+ {{- '[INST]' + message['content'] + '[/INST]' }}
73
+ {%- else %}
74
+ {{- '[INST]' }}
75
+ {%- for block in message['content'] %}
76
+ {%- if block['type'] == 'text' %}
77
+ {{- block['text'] }}
78
+ {%- elif block['type'] == 'image' or block['type'] == 'image_url' %}
79
+ {{- '[IMG]' }}
80
+ {%- else %}
81
+ {{- raise_exception('Only text and image blocks are supported in message content!') }}
82
+ {%- endif %}
83
+ {%- endfor %}
84
+ {{- '[/INST]' }}
85
+ {%- endif %}
86
+ {%- elif message["role"] == "tool_calls" or message.tool_calls is defined %}
87
+ {%- if message.tool_calls is defined %}
88
+ {%- set tool_calls = message.tool_calls %}
89
+ {%- else %}
90
+ {%- set tool_calls = message.content %}
91
+ {%- endif %}
92
+ {{- "[TOOL_CALLS] [" }}
93
+ {%- for tool_call in tool_calls %}
94
+ {%- set out = tool_call.function|tojson %}
95
+ {{- out[:-1] }}
96
+ {%- if not tool_call.id is defined or tool_call.id|length < 9 %}
97
+ {{- raise_exception("Tool call IDs should be alphanumeric strings with length >= 9! (1)" + tool_call.id) }}
98
+ {%- endif %}
99
+ {{- ', "id": "' + tool_call.id[-9:] + '"}' }}
100
+ {%- if not loop.last %}
101
+ {{- ", " }}
102
+ {%- else %}
103
+ {{- "]" + eos_token }}
104
+ {%- endif %}
105
+ {%- endfor %}
106
+ {%- elif message['role'] == 'assistant' %}
107
+ {%- if message['content'] is string %}
108
+ {{- message['content'] + eos_token }}
109
+ {%- else %}
110
+ {{- message['content'][0]['text'] + eos_token }}
111
+ {%- endif %}
112
+ {%- elif message["role"] == "tool_results" or message["role"] == "tool" %}
113
+ {%- if message.content is defined and message.content.content is defined %}
114
+ {%- set content = message.content.content %}
115
+ {%- else %}
116
+ {%- set content = message.content %}
117
+ {%- endif %}
118
+ {{- '[TOOL_RESULTS] {"content": ' + content|string + ", " }}
119
+ {%- if not message.tool_call_id is defined or message.tool_call_id|length < 9 %}
120
+ {{- raise_exception("Tool call IDs should be alphanumeric strings with length >= 9! (2)" + message.tool_call_id) }}
121
+ {%- endif %}
122
+ {{- '"call_id": "' + message.tool_call_id[-9:] + '"}[/TOOL_RESULTS]' }}
123
+ {%- else %}
124
+ {{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}
125
+ {%- endif %}
126
+ {%- endfor %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_mistral_parallel.jinja ADDED
@@ -0,0 +1,93 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if messages[0]["role"] == "system" %}
2
+ {%- set system_message = messages[0]["content"] %}
3
+ {%- set loop_messages = messages[1:] %}
4
+ {%- else %}
5
+ {%- set loop_messages = messages %}
6
+ {%- endif %}
7
+ {%- if not tools is defined %}
8
+ {%- set tools = none %}
9
+ {%- elif tools is not none %}
10
+ {%- set parallel_tool_prompt = "You are a helpful assistant that can call tools. If you call one or more tools, format them in a single JSON array or objects, where each object is a tool call, not as separate objects outside of an array or multiple arrays. Use the format [{\"name\": tool call name, \"arguments\": tool call arguments}, additional tool calls] if you call more than one tool. If you call tools, do not attempt to interpret them or otherwise provide a response until you receive a tool call result that you can interpret for the user." %}
11
+ {%- if system_message is defined %}
12
+ {%- set system_message = parallel_tool_prompt + "\n\n" + system_message %}
13
+ {%- else %}
14
+ {%- set system_message = parallel_tool_prompt %}
15
+ {%- endif %}
16
+ {%- endif %}
17
+ {%- set user_messages = loop_messages | selectattr("role", "equalto", "user") | list %}
18
+
19
+ {%- for message in loop_messages | rejectattr("role", "equalto", "tool") | rejectattr("role", "equalto", "tool_results") | selectattr("tool_calls", "undefined") %}
20
+ {%- if (message["role"] == "user") != (loop.index0 % 2 == 0) %}
21
+ {{- raise_exception("After the optional system message, conversation roles must alternate user/assistant/user/assistant/...") }}
22
+ {%- endif %}
23
+ {%- endfor %}
24
+
25
+ {{- bos_token }}
26
+ {%- for message in loop_messages %}
27
+ {%- if message["role"] == "user" %}
28
+ {%- if tools is not none and (message == user_messages[-1]) %}
29
+ {{- "[AVAILABLE_TOOLS] [" }}
30
+ {%- for tool in tools %}
31
+ {%- set tool = tool.function %}
32
+ {{- '{"type": "function", "function": {' }}
33
+ {%- for key, val in tool.items() if key != "return" %}
34
+ {%- if val is string %}
35
+ {{- '"' + key + '": "' + val + '"' }}
36
+ {%- else %}
37
+ {{- '"' + key + '": ' + val|tojson }}
38
+ {%- endif %}
39
+ {%- if not loop.last %}
40
+ {{- ", " }}
41
+ {%- endif %}
42
+ {%- endfor %}
43
+ {{- "}}" }}
44
+ {%- if not loop.last %}
45
+ {{- ", " }}
46
+ {%- else %}
47
+ {{- "]" }}
48
+ {%- endif %}
49
+ {%- endfor %}
50
+ {{- "[/AVAILABLE_TOOLS]" }}
51
+ {%- endif %}
52
+ {%- if loop.last and system_message is defined %}
53
+ {{- "[INST] " + system_message + "\n\n" + message["content"] + "[/INST]" }}
54
+ {%- else %}
55
+ {{- "[INST] " + message["content"] + "[/INST]" }}
56
+ {%- endif %}
57
+ {%- elif message["role"] == "tool_calls" or message.tool_calls is defined %}
58
+ {%- if message.tool_calls is defined %}
59
+ {%- set tool_calls = message.tool_calls %}
60
+ {%- else %}
61
+ {%- set tool_calls = message.content %}
62
+ {%- endif %}
63
+ {{- "[TOOL_CALLS] [" }}
64
+ {%- for tool_call in tool_calls %}
65
+ {%- set out = tool_call.function|tojson %}
66
+ {{- out[:-1] }}
67
+ {%- if not tool_call.id is defined or tool_call.id|length < 9 %}
68
+ {{- raise_exception("Tool call IDs should be alphanumeric strings with length >= 9! (1)" + tool_call.id) }}
69
+ {%- endif %}
70
+ {{- ', "id": "' + tool_call.id[-9:] + '"}' }}
71
+ {%- if not loop.last %}
72
+ {{- ", " }}
73
+ {%- else %}
74
+ {{- "]" + eos_token }}
75
+ {%- endif %}
76
+ {%- endfor %}
77
+ {%- elif message["role"] == "assistant" %}
78
+ {{- " " + message["content"] + eos_token }}
79
+ {%- elif message["role"] == "tool_results" or message["role"] == "tool" %}
80
+ {%- if message.content is defined and message.content.content is defined %}
81
+ {%- set content = message.content.content %}
82
+ {%- else %}
83
+ {%- set content = message.content %}
84
+ {%- endif %}
85
+ {{- '[TOOL_RESULTS] {"content": ' + content|string + ", " }}
86
+ {%- if not message.tool_call_id is defined or message.tool_call_id|length < 9 %}
87
+ {{- raise_exception("Tool call IDs should be alphanumeric strings with length >= 9! (2)" + message.tool_call_id) }}
88
+ {%- endif %}
89
+ {{- '"call_id": "' + message.tool_call_id[-9:] + '"}[/TOOL_RESULTS]' }}
90
+ {%- else %}
91
+ {{- raise_exception("Only user and assistant roles are supported, with the exception of an initial optional system message!") }}
92
+ {%- endif %}
93
+ {%- endfor %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_phi4_mini.jinja ADDED
@@ -0,0 +1,62 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {%- if messages and messages[0]['role'] == 'system' %}
2
+ {%- set system_message = messages[0]['content']|trim %}
3
+ {%- set messages = messages[1:] %}
4
+ {%- else %}
5
+ {%- set system_message = "You are a helpful assistant." %}
6
+ {%- endif %}
7
+
8
+ {%- if messages %}
9
+ <|system|>
10
+ {{ system_message }}
11
+ {%- if tools %}
12
+ In addition to plain text responses, you can choose to call one or more of the provided functions.
13
+
14
+ Use the following rule to decide when to call a function:
15
+ * if the response can be generated from your internal knowledge (e.g., as in the case of queries like "What is the capital of Poland?"), do so
16
+ * if you need external information that can be obtained by calling one or more of the provided functions, generate a function calls
17
+
18
+ If you decide to call functions:
19
+ * prefix function calls with functools marker (no closing marker required)
20
+ * all function calls should be generated in a single JSON list formatted as functools[{"name": [function name], "arguments": [function arguments as JSON]}, ...]
21
+ * follow the provided JSON schema. Do not hallucinate arguments or values. Do to blindly copy values from the provided samples
22
+ * respect the argument type formatting. E.g., if the type is number and format is float, write value 7 as 7.0
23
+ * make sure you pick the right functions that match the user intent
24
+
25
+
26
+ {%- for t in tools %}
27
+ {{- t | tojson(indent=4) }}
28
+ {{- "\n\n" }}
29
+ {%- endfor %}
30
+ {%- endif %}<|end|>
31
+
32
+ {%- for message in messages %}
33
+ {%- if message.role != "system" %}
34
+ <|{{ message.role }}|>
35
+ {%- if message.content and message.role == "tools" %}
36
+ {"result": {{ message.content }}}
37
+ {%- elif message.content %}
38
+ {{ message.content }}
39
+ {%- elif message.tool_calls %}
40
+ {%- for call in message.tool_calls %}
41
+ {"name": "{{ call.function.name }}", "arguments": {{ call.function.arguments }}}
42
+ {%- if not loop.last %},{% endif %}
43
+ {%- endfor %}
44
+ {%- endif %}<|end|>
45
+ {%- endif %}
46
+ {%- endfor %}<|assistant|>
47
+
48
+ {%- else %}
49
+ {%- if system_message %}
50
+ <|system|>
51
+
52
+ {{ system_message }}<|end|>
53
+ {%- endif %}
54
+ {%- if prompt %}
55
+ <|user|>
56
+
57
+ {{ prompt }}<|end|>
58
+ {%- endif %}<|assistant|>
59
+
60
+ {%- endif %}
61
+ {{ response }}
62
+ {%- if response %}<|user|>{% endif %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_qwen3coder.jinja ADDED
@@ -0,0 +1,117 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {% macro render_extra_keys(json_dict, handled_keys) %}
2
+ {%- if json_dict is mapping %}
3
+ {%- for json_key in json_dict if json_key not in handled_keys %}
4
+ {%- if json_dict[json_key] is mapping or (json_dict[json_key] is sequence and json_dict[json_key] is not string) %}
5
+ {{- '\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | tojson | safe) ~ '</' ~ json_key ~ '>' }}
6
+ {%- else %}
7
+ {{-'\n<' ~ json_key ~ '>' ~ (json_dict[json_key] | string) ~ '</' ~ json_key ~ '>' }}
8
+ {%- endif %}
9
+ {%- endfor %}
10
+ {%- endif %}
11
+ {% endmacro %}
12
+
13
+ {%- if messages[0]["role"] == "system" %}
14
+ {%- set system_message = messages[0]["content"] %}
15
+ {%- set loop_messages = messages[1:] %}
16
+ {%- else %}
17
+ {%- set loop_messages = messages %}
18
+ {%- endif %}
19
+
20
+ {%- if not tools is defined %}
21
+ {%- set tools = [] %}
22
+ {%- endif %}
23
+
24
+ {%- if system_message is defined %}
25
+ {{- "<|im_start|>system\n" + system_message }}
26
+ {%- else %}
27
+ {%- if tools is iterable and tools | length > 0 %}
28
+ {{- "<|im_start|>system\nYou are Qwen, a helpful AI assistant that can interact with a computer to solve tasks." }}
29
+ {%- endif %}
30
+ {%- endif %}
31
+ {%- if tools is iterable and tools | length > 0 %}
32
+ {{- "\n\n# Tools\n\nYou have access to the following functions:\n\n" }}
33
+ {{- "<tools>" }}
34
+ {%- for tool in tools %}
35
+ {%- if tool.function is defined %}
36
+ {%- set tool = tool.function %}
37
+ {%- endif %}
38
+ {{- "\n<function>\n<name>" ~ tool.name ~ "</name>" }}
39
+ {%- if tool.description is defined %}
40
+ {{- '\n<description>' ~ (tool.description | trim) ~ '</description>' }}
41
+ {%- endif %}
42
+ {{- '\n<parameters>' }}
43
+ {%- if tool.parameters is defined and tool.parameters is mapping and tool.parameters.properties is defined and tool.parameters.properties is mapping %}
44
+ {%- for param_name, param_fields in tool.parameters.properties|items %}
45
+ {{- '\n<parameter>' }}
46
+ {{- '\n<name>' ~ param_name ~ '</name>' }}
47
+ {%- if param_fields.type is defined %}
48
+ {{- '\n<type>' ~ (param_fields.type | string) ~ '</type>' }}
49
+ {%- endif %}
50
+ {%- if param_fields.description is defined %}
51
+ {{- '\n<description>' ~ (param_fields.description | trim) ~ '</description>' }}
52
+ {%- endif %}
53
+ {%- set handled_keys = ['name', 'type', 'description'] %}
54
+ {{- render_extra_keys(param_fields, handled_keys) }}
55
+ {{- '\n</parameter>' }}
56
+ {%- endfor %}
57
+ {%- endif %}
58
+ {% set handled_keys = ['type', 'properties'] %}
59
+ {{- render_extra_keys(tool.parameters, handled_keys) }}
60
+ {{- '\n</parameters>' }}
61
+ {%- set handled_keys = ['type', 'name', 'description', 'parameters'] %}
62
+ {{- render_extra_keys(tool, handled_keys) }}
63
+ {{- '\n</function>' }}
64
+ {%- endfor %}
65
+ {{- "\n</tools>" }}
66
+ {{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
67
+ {%- endif %}
68
+ {%- if system_message is defined %}
69
+ {{- '<|im_end|>\n' }}
70
+ {%- else %}
71
+ {%- if tools is iterable and tools | length > 0 %}
72
+ {{- '<|im_end|>\n' }}
73
+ {%- endif %}
74
+ {%- endif %}
75
+ {%- for message in loop_messages %}
76
+ {%- if message.role == "assistant" and message.tool_calls is defined and message.tool_calls is iterable and message.tool_calls | length > 0 %}
77
+ {{- '<|im_start|>' + message.role }}
78
+ {%- if message.content is defined and message.content is string and message.content | trim | length > 0 %}
79
+ {{- '\n' + message.content | trim + '\n' }}
80
+ {%- endif %}
81
+ {%- for tool_call in message.tool_calls %}
82
+ {%- if tool_call.function is defined %}
83
+ {%- set tool_call = tool_call.function %}
84
+ {%- endif %}
85
+ {{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
86
+ {%- if tool_call.arguments is defined %}
87
+ {%- for args_name, args_value in tool_call.arguments|items %}
88
+ {{- '<parameter=' + args_name + '>\n' }}
89
+ {%- set args_value = args_value | tojson | safe if args_value is mapping or (args_value is sequence and args_value is not string) else args_value | string %}
90
+ {{- args_value }}
91
+ {{- '\n</parameter>\n' }}
92
+ {%- endfor %}
93
+ {%- endif %}
94
+ {{- '</function>\n</tool_call>' }}
95
+ {%- endfor %}
96
+ {{- '<|im_end|>\n' }}
97
+ {%- elif message.role == "user" or message.role == "system" or message.role == "assistant" %}
98
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>' + '\n' }}
99
+ {%- elif message.role == "tool" %}
100
+ {%- if loop.previtem and loop.previtem.role != "tool" %}
101
+ {{- '<|im_start|>user\n' }}
102
+ {%- endif %}
103
+ {{- '<tool_response>\n' }}
104
+ {{- message.content }}
105
+ {{- '\n</tool_response>\n' }}
106
+ {%- if not loop.last and loop.nextitem.role != "tool" %}
107
+ {{- '<|im_end|>\n' }}
108
+ {%- elif loop.last %}
109
+ {{- '<|im_end|>\n' }}
110
+ {%- endif %}
111
+ {%- else %}
112
+ {{- '<|im_start|>' + message.role + '\n' + message.content + '<|im_end|>\n' }}
113
+ {%- endif %}
114
+ {%- endfor %}
115
+ {%- if add_generation_prompt %}
116
+ {{- '<|im_start|>assistant\n' }}
117
+ {%- endif %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_toolace.jinja ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token }}
2
+
3
+ {%- if custom_tools is defined %}
4
+ {%- set tools = custom_tools %}
5
+ {%- endif %}
6
+ {%- if not tools is defined %}
7
+ {%- set tools = none %}
8
+ {%- endif %}
9
+
10
+ {#- This block extracts the system message, so we can slot it into the right place. #}
11
+ {%- if messages[0]['role'] == 'system' %}
12
+ {%- set system_message = messages[0]['content']|trim %}
13
+ {%- set messages = messages[1:] %}
14
+ {%- else %}
15
+ {%- set system_message = "You are a helpful assistant with tool calling capabilities. Only reply with a tool call if the function exists in the library provided by the user. If it doesn't exist, just reply directly in natural language." %}
16
+ {%- endif %}
17
+
18
+ {{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
19
+ {%- if tools is not none and not tools_in_user_message %}
20
+ {{- "You are an expert in composing functions. You are given a question and a set of possible functions. Based on the question, you will need to make one or more function/tool calls to achieve the purpose.\n" }}
21
+ {{- "If none of the function can be used, point it out. If the given question lacks the parameters required by the function, also point it out.\n" }}
22
+ {{- "You should only return the function call in tools call sections.\n\n" }}
23
+ {{- "If you decide to invoke any of the function(s), you MUST put it in the format of [func_name1(params_name1=params_value1, params_name2=params_value2...), func_name2(params)]\n" }}
24
+ {{- "You SHOULD NOT include any other text in the response.\n" }}
25
+ {{- "Here is a list of functions in JSON format that you can invoke.\n" }}
26
+ {%- for t in tools %}
27
+ {{- t | tojson(indent=4) }}
28
+ {{- "\n\n" }}
29
+ {%- endfor %}
30
+ {{- "\n" }}
31
+ {%- endif %}
32
+ {{- system_message }}
33
+ {{- "<|eot_id|>" }}
34
+
35
+ {%- for message in messages %}
36
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
37
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
38
+ {%- elif 'tool_calls' in message %}
39
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n[' -}}
40
+ {%- for tool_call in message.tool_calls %}
41
+ {%- if tool_call.function is defined %}
42
+ {%- set tool_call = tool_call.function %}
43
+ {%- endif %}
44
+ {{- tool_call.name + '(' -}}
45
+ {%- for param in tool_call.arguments %}
46
+ {{- param + '=' -}}
47
+ {{- "%s" | format(tool_call.arguments[param]) -}}
48
+ {% if not loop.last %}, {% endif %}
49
+ {%- endfor %}
50
+ {{- ')' -}}
51
+ {% if not loop.last %}, {% endif %}
52
+ {%- endfor %}
53
+ {{- ']<|eot_id|>' -}}
54
+ {%- elif message.role == "tool" or message.role == "ipython" %}
55
+ {{- "<|start_header_id|>ipython<|end_header_id|>\n\n" }}
56
+ {%- if message.content is mapping %}
57
+ {{- message.content | tojson }}
58
+ {%- else %}
59
+ {{- { "output": message.content } | tojson }}
60
+ {%- endif %}
61
+ {{- "<|eot_id|>" }}
62
+ {%- endif %}
63
+ {%- endfor %}
64
+
65
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_xlam_llama.jinja ADDED
@@ -0,0 +1,77 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {{- bos_token }}
2
+ {%- if custom_tools is defined %}
3
+ {%- set tools = custom_tools %}
4
+ {%- endif %}
5
+ {%- if not tools_in_user_message is defined %}
6
+ {%- set tools_in_user_message = true %}
7
+ {%- endif %}
8
+ {%- if not tools is defined %}
9
+ {%- set tools = none %}
10
+ {%- endif %}
11
+
12
+ {#- Extract system message #}
13
+ {{- "<|start_header_id|>system<|end_header_id|>\n\n" }}
14
+ {%- if messages[0]['role'] == 'system' %}
15
+ {%- set system_message = messages[0]['content'] | trim %}
16
+ {%- set messages = messages[1:] %}
17
+ {{- system_message + "\n" }}
18
+ {%- else %}
19
+ {%- set system_message = "You are a helpful assistant. You are developed by Salesforce xLAM team." %}
20
+ {% set format_instruction %}You have access to a set of tools. When using tools, make calls in a single JSON array:
21
+
22
+ [{"name": "tool_call_name", "arguments": {"arg1": "value1", "arg2": "value2"}}, ... (additional parallel tool calls as needed)]
23
+
24
+ If no tool is suitable, state that explicitly. If the user's input lacks required parameters, ask for clarification. Do not interpret or respond until tool results are returned. Once they are available, process them or make additional calls if needed. For tasks that don't require tools, such as casual conversation or general advice, respond directly in plain text. The available tools are:{% endset %}
25
+ {{- system_message + "\n" }}
26
+ {%- if tools is not none %}
27
+ {{- format_instruction + "\n\n" }}
28
+ {%- endif %}
29
+ {%- endif %}
30
+
31
+
32
+ {%- if tools is not none %}
33
+ {%- for t in tools %}
34
+ {{- t | tojson(indent=4) }}
35
+ {{- "\n\n" }}
36
+ {%- endfor %}
37
+ {%- endif %}
38
+ {{- "<|eot_id|>" }}
39
+
40
+ {%- for message in messages %}
41
+ {%- if not (message.role == 'ipython' or message.role == 'tool' or 'tool_calls' in message) %}
42
+ {{- '<|start_header_id|>' + message['role'] + '<|end_header_id|>\n\n'+ message['content'] | trim + '<|eot_id|>' }}
43
+ {%- elif 'tool_calls' in message %}
44
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' -}}
45
+ {%- if message['tool_calls'] %}
46
+ {{- "[" }}
47
+ {%- for tool_call_function in message.tool_calls %}
48
+ {%- set tool_call = tool_call_function.function %}
49
+ {{- '{"name": "' + tool_call.name + '", ' }}
50
+ {{- '"arguments": ' }}
51
+ {{- tool_call.arguments | tojson }}
52
+ {{- "}" }}
53
+ {%- if not loop.last %}
54
+ {{- ", " }}
55
+ {%- endif %}
56
+ {%- endfor %}
57
+ {{- "]" }}
58
+ {{- "<|eot_id|>" }}
59
+ {%- elif message['content'] %}
60
+ {{- message['content'] | trim + '<|eot_id|>' }}
61
+ {%- else %}
62
+ {{- "[]\n" + '<|eot_id|>' }}
63
+ {%- endif %}
64
+ {%- elif message.role == "tool" or message.role == "ipython" %}
65
+ {{- "<|start_header_id|>" + "ipython" + "<|end_header_id|>\n\n" }}
66
+ {%- set content = message["content"] %}
67
+ {%- if content is mapping or (content is iterable and content is not string) %}
68
+ {{- content | tojson }}
69
+ {%- else %}
70
+ {{- content }}
71
+ {%- endif %}
72
+ {{- "<|eot_id|>" }}
73
+ {%- endif %}
74
+ {%- endfor %}
75
+ {%- if add_generation_prompt %}
76
+ {{- '<|start_header_id|>assistant<|end_header_id|>\n\n' }}
77
+ {%- endif %}
asian_ai/performance/vLLM/rust/src/chat/tests/templates/vllm_examples/tool_chat_template_xlam_qwen.jinja ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {# System message #}
2
+ {{- "<|im_start|>system\n" }}
3
+ {%- if messages[0]['role'] == 'system' %}
4
+ {%- set system_message = messages[0]['content'] | trim %}
5
+ {%- set messages = messages[1:] %}
6
+ {{- system_message + "\n" }}
7
+ {%- else %}
8
+ {%- set system_message = "You are a helpful assistant. You are developed by Salesforce xLAM team." %}
9
+ {% set format_instruction %}You have access to a set of tools. When using tools, make calls in a single JSON array:
10
+
11
+ [{"name": "tool_call_name", "arguments": {"arg1": "value1", "arg2": "value2"}}, ... (additional parallel tool calls as needed)]
12
+
13
+ If no tool is suitable, state that explicitly. If the user's input lacks required parameters, ask for clarification. Do not interpret or respond until tool results are returned. Once they are available, process them or make additional calls if needed. For tasks that don't require tools, such as casual conversation or general advice, respond directly in plain text. The available tools are:{% endset %}
14
+ {{- system_message + "\n" }}
15
+ {%- if tools is not none %}
16
+ {{- format_instruction + "\n\n" }}
17
+ {%- endif %}
18
+ {%- endif %}
19
+
20
+ {%- if tools is not none %}
21
+ {%- for func in tools %}
22
+ {{- func | tojson(indent=4) }}
23
+ {{- "\n\n" }}
24
+ {%- endfor %}
25
+ {%- endif %}
26
+ {{- "<|im_end|>\n" }}
27
+ {%- for message in messages %}
28
+ {%- if message['role'] == 'tool' %}
29
+ {{- "<|im_start|>tool\n" }}
30
+ {%- if message.content is defined and message.content.content is defined %}
31
+ {%- set content = message.content.content %}
32
+ {%- else %}
33
+ {%- set content = message.content %}
34
+ {%- endif %}
35
+ {%- if content is mapping or content is iterable and content is not string %}
36
+ {{- content | tojson }}
37
+ {%- else %}
38
+ {{- content }}
39
+ {%- endif %}
40
+ {{- "<|im_end|>\n" }}
41
+ {%- elif 'tool_calls' in message %}
42
+ {{- "<|im_start|>assistant\n" }}
43
+ {%- if message['tool_calls'] %}
44
+ {{- "[" }}
45
+ {%- for tool_call in message.tool_calls %}
46
+ {%- set out = tool_call.function | tojson %}
47
+ {{- out }}
48
+ {%- if not loop.last %}
49
+ {{- ", " }}
50
+ {%- endif %}
51
+ {%- endfor %}
52
+ {{- "]"}}
53
+ {%- elif message['content'] %}
54
+ {{- message['content'] | trim }}
55
+ {%- else %}
56
+ {{- "[]\n" }}
57
+ {%- endif %}
58
+ {{- "<|im_end|>\n" }}
59
+ {%- else %}
60
+ {{- "<|im_start|>" + message['role'] + "\n" + message['content'] | trim + "<|im_end|>\n" }}
61
+ {%- endif %}
62
+ {%- endfor %}
63
+
64
+ {%- if add_generation_prompt %}
65
+ {{- "<|im_start|>assistant\n" }}
66
+ {%- endif %}
asian_ai/performance/vLLM/rust/src/cmd/Cargo.toml ADDED
@@ -0,0 +1,41 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [package]
2
+ name = "vllm-cmd"
3
+ version.workspace = true
4
+ edition.workspace = true
5
+ license.workspace = true
6
+
7
+ [[bin]]
8
+ name = "vllm-rs"
9
+ path = "src/main.rs"
10
+
11
+ [features]
12
+ default = []
13
+ native-tls-vendored = ["dep:native-tls-vendored"]
14
+
15
+ [dependencies]
16
+ anyhow.workspace = true
17
+ clap.workspace = true
18
+ educe.workspace = true
19
+ itertools.workspace = true
20
+ mimalloc.workspace = true
21
+ native-tls-vendored = { workspace = true, optional = true }
22
+ serde.workspace = true
23
+ serde_json.workspace = true
24
+ serde_with.workspace = true
25
+ thiserror-ext.workspace = true
26
+ time.workspace = true
27
+ tokio = { workspace = true, features = ["signal"] }
28
+ tokio-util.workspace = true
29
+ tracing.workspace = true
30
+ tracing-subscriber.workspace = true
31
+ uuid.workspace = true
32
+ vllm-chat.workspace = true
33
+ vllm-engine-core-client.workspace = true
34
+ vllm-managed-engine.workspace = true
35
+ vllm-server.workspace = true
36
+
37
+ [dev-dependencies]
38
+ expect-test.workspace = true
39
+
40
+ [lints]
41
+ workspace = true
asian_ai/performance/vLLM/rust/src/cmd/examples/README.md ADDED
@@ -0,0 +1,44 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # `vllm-rs` CLI Quick Start
2
+
3
+ Start Qwen3 with one managed `vllm-rs serve` command from the repo root:
4
+
5
+ ```bash
6
+ HF_HUB_OFFLINE=1 \
7
+ VLLM_CPU_KVCACHE_SPACE=2 \
8
+ VLLM_HOST_IP=127.0.0.1 \
9
+ VLLM_LOOPBACK_IP=127.0.0.1 \
10
+ cargo run --bin vllm-rs -- serve \
11
+ Qwen/Qwen3-0.6B \
12
+ --python ../vllm/.venv/bin/python \
13
+ --max-model-len 512 \
14
+ -- \
15
+ --dtype float16
16
+ ```
17
+
18
+ This launches:
19
+
20
+ - a managed headless Python `vllm` engine
21
+ - the Rust OpenAI-compatible frontend on `127.0.0.1:8000`
22
+
23
+ All Python engine arguments must be placed after `--`. Arguments before `--` are parsed by the Rust
24
+ frontend itself.
25
+
26
+ You can then send OpenAI-style requests to the Rust frontend:
27
+
28
+ ```bash
29
+ curl http://127.0.0.1:8000/v1/chat/completions \
30
+ -H "Content-Type: application/json" \
31
+ -d '{
32
+ "model": "Qwen/Qwen3-0.6B",
33
+ "messages": [{"role": "user", "content": "What is the capital of France?"}],
34
+ "stream": true
35
+ }'
36
+ ```
37
+
38
+ If you already started headless `vllm` yourself, use `frontend` instead:
39
+
40
+ ```bash
41
+ cargo run --bin vllm-rs -- frontend \
42
+ --handshake-address tcp://127.0.0.1:62100 \
43
+ Qwen/Qwen3-0.6B
44
+ ```
asian_ai/performance/vLLM/rust/src/cmd/src/cli.rs ADDED
@@ -0,0 +1,694 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ //! CLI argument definitions for the `vllm-rs` binary.
2
+ //!
3
+ //! Python vLLM references:
4
+ //! - Engine args: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/engine/arg_utils.py#L657-L1311>
5
+ //! - Environment variables: <https://github.com/vllm-project/vllm/blob/bc2c0c86efb28e77677a3cfb8687e976914a313a/vllm/envs.py#L472>
6
+
7
+ mod unsupported;
8
+
9
+ use std::collections::HashMap;
10
+ use std::ffi::{OsStr, OsString};
11
+ use std::path::PathBuf;
12
+ use std::time::Duration;
13
+
14
+ use clap::{Args, Parser, Subcommand};
15
+ use educe::Educe;
16
+ use serde::de::DeserializeOwned;
17
+ use serde::{Deserialize, Serialize};
18
+ use serde_json::Value;
19
+ use serde_with::{DefaultOnNull, OneOrMany, serde_as};
20
+ use thiserror_ext::AsReport as _;
21
+ use uuid::Uuid;
22
+ use vllm_chat::ReasoningParserFactory;
23
+ use vllm_engine_core_client::TransportMode;
24
+ use vllm_managed_engine::ManagedEngineConfig;
25
+ use vllm_managed_engine::cli::{ManagedEngineArgs, repartition_managed_engine_args};
26
+ use vllm_server::{
27
+ ApiServerOptions, ChatTemplateContentFormatOption, Config, CoordinatorMode, CorsConfig,
28
+ DEFAULT_KEEP_ALIVE_TIMEOUT, HttpListenerMode, ParserSelection, RendererSelection, TlsConfig,
29
+ };
30
+
31
+ use crate::cli::unsupported::UnsupportedArgs;
32
+
33
+ /// Top-level parser for the `vllm-rs` binary.
34
+ #[derive(Debug, Parser)]
35
+ #[command(
36
+ name = "vllm-rs",
37
+ about = "Rust frontend and managed-engine CLI for vLLM."
38
+ )]
39
+ pub struct Cli {
40
+ #[command(subcommand)]
41
+ pub command: Command,
42
+ }
43
+
44
+ impl Cli {
45
+ pub fn parse() -> Self {
46
+ Self::try_parse_from(std::env::args_os()).unwrap_or_else(|error| error.exit())
47
+ }
48
+
49
+ pub fn try_parse_from<I, T>(itr: I) -> Result<Self, clap::Error>
50
+ where
51
+ I: IntoIterator<Item = T>,
52
+ T: Into<OsString>,
53
+ {
54
+ let args: Vec<OsString> = itr.into_iter().map(Into::into).collect();
55
+ let repartitioned_args = repartition_managed_engine_args::<Self>(&args, Some("serve"))?;
56
+ <Self as Parser>::try_parse_from(&repartitioned_args).inspect(|cli| {
57
+ if let Command::Serve(serve) = &cli.command
58
+ && serve.debug_cli
59
+ {
60
+ println!(
61
+ "Original CLI args: {}\n",
62
+ args.join(OsStr::new(" ")).display()
63
+ );
64
+ println!(
65
+ "Repartitioned CLI args: {}\n",
66
+ repartitioned_args.join(OsStr::new(" ")).display()
67
+ );
68
+ println!(
69
+ "Passthrough Python args: {}",
70
+ serve.managed_engine.python_args.join(" ")
71
+ );
72
+ std::process::exit(0);
73
+ }
74
+ })
75
+ }
76
+ }
77
+
78
+ /// Supported top-level CLI commands.
79
+ #[derive(Debug, Subcommand, PartialEq, Eq)]
80
+ pub enum Command {
81
+ /// Run the Rust OpenAI frontend as a Python-supervised worker.
82
+ Frontend(FrontendArgs),
83
+ /// Launch a managed Python headless engine, then run the Rust OpenAI
84
+ /// frontend.
85
+ Serve(ServeArgs),
86
+ }
87
+
88
+ /// A JSON-encoded list of strings, matching Python's `json.loads` CLI type for
89
+ /// the CORS list arguments (e.g. `--allowed-origins '["*"]'`). Parsing the whole
90
+ /// value as one item keeps clap from treating the field as a repeated flag.
91
+ #[derive(Clone, Debug, PartialEq, Eq, Deserialize)]
92
+ #[serde(transparent)]
93
+ pub struct JsonStringList(pub Vec<String>);
94
+
95
+ /// Runtime arguments shared by both paths of the Rust frontend:
96
+ ///
97
+ /// - External-engine mode: Python-supervised bootstrap, `vllm serve` -> `vllm-rs frontend`.
98
+ /// Arguments are deserialized from a single JSON object and defaults follow `serde` attrs.
99
+ /// - Managed-engine mode: Rust-managed Python engine, `vllm-rs serve`.
100
+ /// Arguments are parsed from CLI flags and defaults follow `clap` attrs.
101
+ #[serde_as]
102
+ #[derive(Educe, Clone, Args, PartialEq, Eq, Deserialize)]
103
+ #[educe(Debug)]
104
+ pub struct SharedRuntimeArgs {
105
+ #[serde(rename = "model_tag")]
106
+ /// Model identifier or local model directory used for backend loading and
107
+ /// public model ID.
108
+ pub model: String,
109
+
110
+ /// Maximum time to wait for the expected engines to register on the
111
+ /// frontend transport.
112
+ #[arg(
113
+ long = "engine-ready-timeout-secs",
114
+ env = "VLLM_ENGINE_READY_TIMEOUT_S",
115
+ default_value_t = default_engine_ready_timeout_secs()
116
+ )]
117
+ #[serde(default = "default_engine_ready_timeout_secs")]
118
+ pub engine_ready_timeout_secs: u64,
119
+
120
+ /// Select the tool call parser depending on the model that you're using.
121
+ /// Use `auto` to infer from the model or `none` to disable parsing.
122
+ #[arg(long, default_value_t)]
123
+ #[serde(default = "default_py_bootstrap_parser_selection")]
124
+ pub tool_call_parser: ParserSelection,
125
+ /// Select the reasoning parser depending on the model that you're using.
126
+ /// Use `auto` to infer from the model or `none` to disable parsing.
127
+ #[arg(long, default_value_t)]
128
+ #[serde(default = "default_py_bootstrap_parser_selection")]
129
+ pub reasoning_parser: ParserSelection,
130
+ /// Select the chat renderer implementation.
131
+ #[arg(long = "tokenizer-mode", default_value_t)]
132
+ #[serde(default, rename = "tokenizer_mode")]
133
+ pub renderer: RendererSelection,
134
+ /// Disable multimodal inputs and treat the model as language-only.
135
+ #[arg(long)]
136
+ #[serde(default)]
137
+ pub language_model_only: bool,
138
+ /// Override the maximum model context length. When set, the frontend uses
139
+ /// this value instead of the model's `max_position_embeddings` from
140
+ /// `config.json`.
141
+ #[arg(long)]
142
+ pub max_model_len: Option<u32>,
143
+ /// Maximum number of log probabilities to return when `logprobs` is
144
+ /// specified in sampling parameters. `-1` means no cap.
145
+ #[arg(long, value_parser = clap::value_parser!(i32).range(-1..), allow_negative_numbers = true)]
146
+ #[serde(default)]
147
+ pub max_logprobs: Option<i32>,
148
+ /// TCP port for the gRPC Generate service. When not set, no gRPC server is
149
+ /// started.
150
+ #[arg(long)]
151
+ #[serde(default)]
152
+ pub grpc_port: Option<u16>,
153
+ /// Maximum time to wait for active requests to drain during shutdown.
154
+ #[arg(long, default_value_t = 0)]
155
+ #[serde(default)]
156
+ pub shutdown_timeout: u64,
157
+ /// Maximum idle time (seconds) on a keep-alive HTTP connection before the
158
+ /// server closes it (default 5).
159
+ #[arg(long = "http-timeout-keep-alive", env = "VLLM_HTTP_TIMEOUT_KEEP_ALIVE")]
160
+ #[serde(default)]
161
+ pub http_timeout_keep_alive: Option<u64>,
162
+
163
+ /// The file path to the chat template, or the template in single-line form
164
+ /// for the specified model.
165
+ #[arg(long)]
166
+ #[serde(default)]
167
+ pub chat_template: Option<String>,
168
+
169
+ /// Default keyword arguments to pass to the chat template renderer.
170
+ ///
171
+ /// These will be merged with request-level chat_template_kwargs, with
172
+ /// request values taking precedence. Useful for setting default
173
+ /// behavior for reasoning models.
174
+ ///
175
+ /// Example: `{"enable_thinking": false}` to disable thinking mode by
176
+ /// default for Qwen3/DeepSeek models.
177
+ #[arg(long, value_parser = parse_json::<HashMap<String, Value>>, value_name = "JSON")]
178
+ #[serde(default)]
179
+ pub default_chat_template_kwargs: Option<HashMap<String, Value>>,
180
+
181
+ /// The format to render message content within a chat template.
182
+ ///
183
+ /// * "auto" detects the format from the template
184
+ /// * "string" renders content as a string. Example: `"Hello World"`
185
+ /// * "openai" renders content as a list of dictionaries, similar to OpenAI schema. Example:
186
+ /// `[{"type": "text", "text": "Hello world!"}]`
187
+ #[arg(long, default_value_t)]
188
+ #[serde(default)]
189
+ pub chat_template_content_format: ChatTemplateContentFormatOption,
190
+
191
+ /// Log a summary line for each completed request, including prompt/output
192
+ /// token counts and finish reason.
193
+ #[arg(long)]
194
+ #[serde(default)]
195
+ pub enable_log_requests: bool,
196
+
197
+ /// Include prompt_tokens_details in usage when cached prompt tokens are
198
+ /// present.
199
+ #[arg(
200
+ long,
201
+ default_missing_value = "true",
202
+ num_args = 0..=1
203
+ )]
204
+ #[serde(default)]
205
+ pub enable_prompt_tokens_details: bool,
206
+
207
+ /// If specified, API server will add X-Request-Id header to responses.
208
+ #[arg(
209
+ long,
210
+ default_missing_value = "true",
211
+ num_args = 0..=1
212
+ )]
213
+ #[serde(default)]
214
+ pub enable_request_id_headers: bool,
215
+
216
+ /// If provided, the server will require one of these keys to be presented
217
+ /// in the Authorization header.
218
+ #[educe(Debug(ignore))]
219
+ #[arg(long, env = "VLLM_API_KEY", value_delimiter = ' ')]
220
+ #[serde_as(as = "DefaultOnNull<OneOrMany<_>>")]
221
+ #[serde(default)]
222
+ pub api_key: Vec<String>,
223
+
224
+ /// Disable periodic logging of engine statistics (throughput, queue depth,
225
+ /// cache usage).
226
+ #[arg(long)]
227
+ #[serde(default)]
228
+ pub disable_log_stats: bool,
229
+
230
+ /// The model name(s) used in the API. If multiple names are provided, the
231
+ /// server will respond to any of the provided names. The model name in the
232
+ /// model field of a response will be the first name in this list. If not
233
+ /// specified, the model name will be the same as the `--model` argument.
234
+ /// Noted that this name(s) will also be used in `model_name` tag
235
+ /// content of prometheus metrics, if multiple names provided, metrics
236
+ /// tag will take the first one.
237
+ #[arg(long, num_args = 0..)]
238
+ #[serde(default)]
239
+ pub served_model_name: Vec<String>,
240
+
241
+ /// CORS allowed origins as a JSON list. `["*"]` allows any origin.
242
+ #[arg(long, value_parser = parse_json::<JsonStringList>, value_name = "JSON", default_value = r#"["*"]"#)]
243
+ #[serde(default = "default_cors_wildcard")]
244
+ pub allowed_origins: JsonStringList,
245
+
246
+ /// CORS allowed methods as a JSON list. `["*"]` allows the standard set.
247
+ #[arg(long, value_parser = parse_json::<JsonStringList>, value_name = "JSON", default_value = r#"["*"]"#)]
248
+ #[serde(default = "default_cors_wildcard")]
249
+ pub allowed_methods: JsonStringList,
250
+
251
+ /// CORS allowed request headers as a JSON list. `["*"]` mirrors the request.
252
+ #[arg(long, value_parser = parse_json::<JsonStringList>, value_name = "JSON", default_value = r#"["*"]"#)]
253
+ #[serde(default = "default_cors_wildcard")]
254
+ pub allowed_headers: JsonStringList,
255
+
256
+ /// Allow CORS credentials (cookies, authorization headers).
257
+ #[arg(
258
+ long,
259
+ default_missing_value = "true",
260
+ num_args = 0..=1
261
+ )]
262
+ #[serde(default)]
263
+ pub allow_credentials: bool,
264
+
265
+ /// The file path to the SSL key file. When omitted, the key is read from
266
+ /// `--ssl-certfile` (combined PEM).
267
+ #[arg(long)]
268
+ #[serde(default)]
269
+ pub ssl_keyfile: Option<String>,
270
+
271
+ /// The file path to the SSL cert file. Enables TLS when set.
272
+ #[arg(long)]
273
+ #[serde(default)]
274
+ pub ssl_certfile: Option<String>,
275
+
276
+ /// The CA certificates file used to verify client certificates (mTLS).
277
+ #[arg(long)]
278
+ #[serde(default)]
279
+ pub ssl_ca_certs: Option<String>,
280
+
281
+ /// Whether a client certificate is required: 0 = none, 1 = optional,
282
+ /// 2 = required (mirrors Python's `ssl.CERT_*`).
283
+ #[arg(long, default_value_t = 0, value_parser = clap::value_parser!(i32).range(0..=2))]
284
+ #[serde(default)]
285
+ pub ssl_cert_reqs: i32,
286
+
287
+ /// OpenSSL cipher string for HTTPS (TLS 1.2 and below).
288
+ /// When unset, the linked OpenSSL's default suites are used.
289
+ #[arg(long)]
290
+ #[serde(default)]
291
+ pub ssl_ciphers: Option<String>,
292
+
293
+ /// Profiler configuration forwarded by the Python supervisor.
294
+ ///
295
+ /// When set with a non-null `profiler` type, the Rust frontend registers
296
+ /// the `/start_profile` and `/stop_profile` routes and forwards calls to
297
+ /// the engine via the `"profile"` utility RPC.
298
+ #[arg(long, value_parser = parse_json::<ProfilerConfig>, value_name = "JSON")]
299
+ #[serde(default)]
300
+ pub profiler_config: Option<ProfilerConfig>,
301
+
302
+ /// Unsupported Python vLLM frontend arguments recognized but not yet
303
+ /// implemented in Rust.
304
+ #[educe(Debug(ignore))]
305
+ #[command(flatten)]
306
+ #[serde(default, flatten)]
307
+ pub unsupported: UnsupportedArgs,
308
+ }
309
+
310
+ impl SharedRuntimeArgs {
311
+ /// Maximum time to wait for the expected engines to register on the
312
+ /// frontend transport.
313
+ pub fn ready_timeout(&self) -> Duration {
314
+ Duration::from_secs(self.engine_ready_timeout_secs)
315
+ }
316
+
317
+ /// Maximum time to wait for active requests to drain during shutdown.
318
+ pub fn shutdown_timeout(&self) -> Duration {
319
+ Duration::from_secs(self.shutdown_timeout)
320
+ }
321
+
322
+ /// Maximum idle time on a keep-alive HTTP connection before the server
323
+ /// closes it.
324
+ pub fn keep_alive_timeout(&self) -> Duration {
325
+ self.http_timeout_keep_alive
326
+ .map_or(DEFAULT_KEEP_ALIVE_TIMEOUT, Duration::from_secs)
327
+ }
328
+
329
+ /// Return the configured profiler mode, when profiling is enabled.
330
+ pub fn profiler(&self) -> Option<String> {
331
+ self.profiler_config.as_ref().and_then(|c| c.profiler.clone())
332
+ }
333
+
334
+ /// Return the profiler config JSON for managed Python engine forwarding.
335
+ pub fn profiler_config_json(&self) -> Option<String> {
336
+ self.profiler_config
337
+ .as_ref()
338
+ .map(serde_json::to_string)
339
+ .transpose()
340
+ .expect("profiler config serialization should not fail")
341
+ }
342
+
343
+ /// Apply fallback logic for API key configuration from env variables.
344
+ fn apply_env_api_key_fallback(&mut self) {
345
+ if self.api_key.is_empty()
346
+ && let Ok(api_key) = std::env::var("VLLM_API_KEY")
347
+ {
348
+ self.api_key.push(api_key);
349
+ }
350
+ }
351
+
352
+ /// Build the OpenAI-server config for the Python-bootstrap worker contract.
353
+ ///
354
+ /// The resulting config binds the Python-supplied transport addresses and
355
+ /// inherits an already open HTTP listener from the supervisor process.
356
+ fn into_bootstrapped_config(
357
+ self,
358
+ listen_fd: i32,
359
+ input_address: String,
360
+ output_address: String,
361
+ coordinator_address: Option<String>,
362
+ engine_start_index: u32,
363
+ engine_count: usize,
364
+ ) -> Config {
365
+ let ready_timeout = self.ready_timeout();
366
+ let shutdown_timeout = self.shutdown_timeout();
367
+ let keep_alive_timeout = self.keep_alive_timeout();
368
+ let api_server_options = self.api_server_options();
369
+ let cors = self.cors_config();
370
+ let tls = self.tls_config();
371
+ let profiler = self.profiler();
372
+
373
+ Config {
374
+ transport_mode: TransportMode::Bootstrapped {
375
+ input_address,
376
+ output_address,
377
+ engine_start_index,
378
+ engine_count,
379
+ ready_timeout,
380
+ },
381
+ coordinator_mode: match coordinator_address {
382
+ Some(address) => CoordinatorMode::External { address },
383
+ None => CoordinatorMode::None,
384
+ },
385
+ model: self.model,
386
+ served_model_name: self.served_model_name,
387
+ listener_mode: HttpListenerMode::InheritedFd { fd: listen_fd },
388
+ tool_call_parser: self.tool_call_parser,
389
+ reasoning_parser: self.reasoning_parser,
390
+ renderer: self.renderer,
391
+ language_model_only: self.language_model_only,
392
+ chat_template: self.chat_template,
393
+ default_chat_template_kwargs: self.default_chat_template_kwargs,
394
+ chat_template_content_format: self.chat_template_content_format,
395
+ max_logprobs: self.max_logprobs,
396
+ api_server_options,
397
+ cors,
398
+ tls,
399
+ api_keys: self.api_key,
400
+ disable_log_stats: self.disable_log_stats,
401
+ grpc_port: self.grpc_port,
402
+ shutdown_timeout,
403
+ keep_alive_timeout,
404
+ profiler,
405
+ }
406
+ }
407
+
408
+ /// Build the OpenAI-server config for the managed `serve` path that still
409
+ /// owns the startup handshake and binds its own HTTP listener.
410
+ fn into_managed_config(
411
+ self,
412
+ listener_mode: HttpListenerMode,
413
+ handshake_address: String,
414
+ advertised_host: String,
415
+ engine_count: usize,
416
+ local_input_address: Option<String>,
417
+ local_output_address: Option<String>,
418
+ ) -> Config {
419
+ let ready_timeout = self.ready_timeout();
420
+ let shutdown_timeout = self.shutdown_timeout();
421
+ let keep_alive_timeout = self.keep_alive_timeout();
422
+ let api_server_options = self.api_server_options();
423
+ let cors = self.cors_config();
424
+ let tls = self.tls_config();
425
+ let profiler = self.profiler();
426
+
427
+ Config {
428
+ transport_mode: TransportMode::HandshakeOwner {
429
+ handshake_address,
430
+ advertised_host,
431
+ engine_count,
432
+ ready_timeout,
433
+ local_input_address,
434
+ local_output_address,
435
+ },
436
+ coordinator_mode: CoordinatorMode::MaybeInProc,
437
+ model: self.model,
438
+ served_model_name: self.served_model_name,
439
+ listener_mode,
440
+ tool_call_parser: self.tool_call_parser,
441
+ reasoning_parser: self.reasoning_parser,
442
+ renderer: self.renderer,
443
+ language_model_only: self.language_model_only,
444
+ chat_template: self.chat_template,
445
+ default_chat_template_kwargs: self.default_chat_template_kwargs,
446
+ chat_template_content_format: self.chat_template_content_format,
447
+ max_logprobs: self.max_logprobs,
448
+ api_server_options,
449
+ cors,
450
+ tls,
451
+ api_keys: self.api_key,
452
+ disable_log_stats: self.disable_log_stats,
453
+ grpc_port: self.grpc_port,
454
+ shutdown_timeout,
455
+ keep_alive_timeout,
456
+ profiler,
457
+ }
458
+ }
459
+
460
+ fn api_server_options(&self) -> ApiServerOptions {
461
+ ApiServerOptions {
462
+ enable_log_requests: self.enable_log_requests,
463
+ enable_prompt_tokens_details: self.enable_prompt_tokens_details,
464
+ enable_request_id_headers: self.enable_request_id_headers,
465
+ }
466
+ }
467
+
468
+ fn cors_config(&self) -> CorsConfig {
469
+ CorsConfig {
470
+ allow_origins: self.allowed_origins.0.clone(),
471
+ allow_methods: self.allowed_methods.0.clone(),
472
+ allow_headers: self.allowed_headers.0.clone(),
473
+ allow_credentials: self.allow_credentials,
474
+ }
475
+ }
476
+
477
+ /// Build the TLS config: `Some` when any `ssl_*` argument is set, else
478
+ /// `None` (plaintext). The combination is validated in [`Config::validate`].
479
+ fn tls_config(&self) -> Option<TlsConfig> {
480
+ let tls_requested = self.ssl_certfile.is_some()
481
+ || self.ssl_keyfile.is_some()
482
+ || self.ssl_ca_certs.is_some()
483
+ || self.ssl_cert_reqs != 0
484
+ || self.ssl_ciphers.is_some();
485
+ tls_requested.then(|| TlsConfig {
486
+ cert_file: self.ssl_certfile.clone(),
487
+ key_file: self.ssl_keyfile.clone(),
488
+ ca_certs: self.ssl_ca_certs.clone(),
489
+ cert_reqs: self.ssl_cert_reqs,
490
+ ciphers: self.ssl_ciphers.clone(),
491
+ })
492
+ }
493
+ }
494
+
495
+ fn default_engine_ready_timeout_secs() -> u64 {
496
+ 600
497
+ }
498
+
499
+ fn default_cors_wildcard() -> JsonStringList {
500
+ JsonStringList(vec!["*".to_string()])
501
+ }
502
+
503
+ fn default_py_bootstrap_parser_selection() -> ParserSelection {
504
+ ParserSelection::None
505
+ }
506
+
507
+ /// Minimal profiler configuration parsed from `--profiler-config`.
508
+ ///
509
+ /// Only the `profiler` field is inspected by the Rust frontend to decide
510
+ /// whether to register the `/start_profile` and `/stop_profile` routes.
511
+ /// All other fields are accepted but ignored — they are consumed by the
512
+ /// Python engine layer.
513
+ #[derive(Clone, Debug, Default, PartialEq, Eq, Deserialize, Serialize)]
514
+ pub struct ProfilerConfig {
515
+ /// Profiler backend type (e.g. `"torch"`, `"cuda"`). When `null` or
516
+ /// absent, profiling is disabled.
517
+ #[serde(default)]
518
+ pub profiler: Option<String>,
519
+ /// Additional Python profiler config fields consumed by the engine layer.
520
+ #[serde(flatten)]
521
+ pub extra: serde_json::Map<String, Value>,
522
+ }
523
+
524
+ fn parse_json<T: DeserializeOwned>(value: &str) -> Result<T, String> {
525
+ serde_json::from_str(value).map_err(|e| format!("invalid JSON object: {}", e.as_report()))
526
+ }
527
+
528
+ fn parse_runtime_args_json(value: &str) -> Result<SharedRuntimeArgs, String> {
529
+ let mut args: SharedRuntimeArgs = serde_json::from_str(value)
530
+ .map_err(|e| format!("invalid JSON arguments: {}", e.as_report()))?;
531
+ // --args-json is parsed with serde, so clap's env support does not run for
532
+ // the Python-supervised frontend path.
533
+ args.apply_env_api_key_fallback();
534
+ args.unsupported.check()?;
535
+ Ok(args)
536
+ }
537
+
538
+ /// Arguments for running the Rust frontend as a Python-bootstrapped worker.
539
+ #[derive(Educe, Clone, Args, PartialEq, Eq)]
540
+ #[educe(Debug)]
541
+ pub struct FrontendArgs {
542
+ /// Inherited listening socket file descriptor passed by the Python
543
+ /// supervisor.
544
+ #[arg(long)]
545
+ pub listen_fd: i32,
546
+ /// Frontend input ROUTER socket address that the Python engines will
547
+ /// connect to.
548
+ #[arg(long)]
549
+ pub input_address: String,
550
+ /// Frontend output PULL socket address that the Python engines will push
551
+ /// responses to.
552
+ #[arg(long)]
553
+ pub output_address: String,
554
+ /// Optional Python-owned frontend-side DP coordinator socket address for
555
+ /// external coordinator mode in the bootstrapped frontend path, i.e.,
556
+ /// `stats_update_address`.
557
+ #[arg(long)]
558
+ pub coordinator_address: Option<String>,
559
+ /// First data-parallel engine rank expected to register with this
560
+ /// bootstrapped frontend.
561
+ #[arg(long, default_value_t = 0)]
562
+ pub engine_start_index: u32,
563
+ /// Total number of data-parallel engines expected for this frontend.
564
+ #[arg(long, default_value_t = 1)]
565
+ pub engine_count: usize,
566
+
567
+ /// Shared frontend arguments as one JSON object.
568
+ #[arg(long = "args-json", value_parser = parse_runtime_args_json, value_name = "JSON")]
569
+ pub runtime: SharedRuntimeArgs,
570
+ }
571
+
572
+ impl FrontendArgs {
573
+ /// Convert the CLI arguments into the OpenAI server's runtime config.
574
+ pub fn into_config(self) -> Config {
575
+ self.runtime.into_bootstrapped_config(
576
+ self.listen_fd,
577
+ self.input_address,
578
+ self.output_address,
579
+ self.coordinator_address,
580
+ self.engine_start_index,
581
+ self.engine_count,
582
+ )
583
+ }
584
+ }
585
+
586
+ /// Arguments for the managed-engine mode that spawns Python on behalf of the
587
+ /// user.
588
+ #[derive(Educe, Clone, Args, PartialEq, Eq)]
589
+ #[educe(Debug)]
590
+ #[command(override_usage = "vllm-rs serve <MODEL> [OPTIONS] [-- <PYTHON_ARGS>...]")]
591
+ pub struct ServeArgs {
592
+ /// Only launch the managed Python headless engine and do not start the Rust
593
+ /// frontend.
594
+ #[arg(long)]
595
+ pub headless: bool,
596
+ /// HTTP bind host for the OpenAI-compatible server.
597
+ #[arg(long, default_value = "127.0.0.1")]
598
+ pub host: String,
599
+ /// HTTP bind port for the OpenAI-compatible server.
600
+ #[arg(long, default_value_t = 8000)]
601
+ pub port: u16,
602
+ /// Unix domain socket path. If set, host and port arguments are ignored.
603
+ #[arg(long)]
604
+ pub uds: Option<String>,
605
+
606
+ /// Flag to print debug information about CLI argument parsing and exit.
607
+ #[educe(Debug(ignore))]
608
+ #[arg(long, hide = true, env = "VLLM_RS_DEBUG_CLI")]
609
+ pub debug_cli: bool,
610
+
611
+ /// Shared frontend arguments.
612
+ #[command(flatten)]
613
+ pub runtime: SharedRuntimeArgs,
614
+
615
+ /// Managed Python headless-engine arguments.
616
+ #[command(flatten)]
617
+ pub managed_engine: ManagedEngineArgs,
618
+ }
619
+
620
+ impl ServeArgs {
621
+ /// Build the OpenAI-server runtime config used after the managed Python
622
+ /// engine starts.
623
+ pub fn to_frontend_config(&self, handshake_address: String) -> Config {
624
+ // Prefer IPC sockets for local engine input/output.
625
+ let (local_input_address, local_output_address) =
626
+ self.managed_engine.frontend_local_only().then(frontend_ipc_addresses).unzip();
627
+ let listener_mode = match &self.uds {
628
+ Some(path) => HttpListenerMode::BindUnix { path: path.clone() },
629
+ None => HttpListenerMode::BindTcp {
630
+ host: self.host.clone(),
631
+ port: self.port,
632
+ },
633
+ };
634
+
635
+ self.runtime.clone().into_managed_config(
636
+ listener_mode,
637
+ handshake_address,
638
+ self.managed_engine.handshake_host.clone(),
639
+ self.managed_engine.data_parallel_size,
640
+ local_input_address,
641
+ local_output_address,
642
+ )
643
+ }
644
+
645
+ /// Build the managed Python-engine spawn configuration with the given
646
+ /// handshake port.
647
+ pub fn to_managed_engine_config(&self, handshake_port: u16) -> ManagedEngineConfig {
648
+ let reasoning_parser =
649
+ effective_engine_reasoning_parser(&self.runtime.reasoning_parser, &self.runtime.model);
650
+ let profiler_config = self.runtime.profiler_config_json();
651
+
652
+ self.managed_engine.clone().into_config(
653
+ self.runtime.model.clone(),
654
+ self.runtime.max_model_len,
655
+ self.runtime.max_logprobs,
656
+ profiler_config,
657
+ reasoning_parser.as_deref(),
658
+ self.runtime.language_model_only,
659
+ self.runtime.disable_log_stats,
660
+ self.runtime.shutdown_timeout,
661
+ handshake_port,
662
+ )
663
+ }
664
+ }
665
+
666
+ fn effective_engine_reasoning_parser(selection: &ParserSelection, model: &str) -> Option<String> {
667
+ match selection {
668
+ ParserSelection::Auto => ReasoningParserFactory::global()
669
+ .resolve_name_for_model(model)
670
+ .map(str::to_string),
671
+ ParserSelection::None => None,
672
+ ParserSelection::Explicit(name) => Some(name.clone()),
673
+ }
674
+ }
675
+
676
+ /// Allocate fresh IPC endpoints for one managed frontend instance.
677
+ fn frontend_ipc_addresses() -> (String, String) {
678
+ let preferred_base_path = std::env::var_os("VLLM_RPC_BASE_PATH")
679
+ .map(PathBuf::from)
680
+ .unwrap_or_else(std::env::temp_dir);
681
+ let input_name = format!("vllm-rs-i-{}", Uuid::new_v4().simple());
682
+ let output_name = format!("vllm-rs-o-{}", Uuid::new_v4().simple());
683
+
684
+ let input = preferred_base_path.join(input_name);
685
+ let output = preferred_base_path.join(output_name);
686
+
687
+ (
688
+ format!("ipc://{}", input.to_string_lossy()),
689
+ format!("ipc://{}", output.to_string_lossy()),
690
+ )
691
+ }
692
+
693
+ #[cfg(test)]
694
+ mod tests;
asian_ai/performance/vLLM/rust/src/cmd/src/cli/tests.rs ADDED
@@ -0,0 +1,1718 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use expect_test::expect;
2
+ use vllm_engine_core_client::TransportMode;
3
+ use vllm_server::{Config, HttpListenerMode, ParserSelection, RendererSelection};
4
+
5
+ use super::{Cli, Command};
6
+
7
+ #[test]
8
+ fn serve_args_forward_python_flags_with_separator() {
9
+ let cli = Cli::try_parse_from([
10
+ "vllm-rs",
11
+ "serve",
12
+ "Qwen/Qwen3-0.6B",
13
+ "--python",
14
+ "../vllm/.venv/bin/python",
15
+ "--max-model-len",
16
+ "512",
17
+ "--",
18
+ "--dtype",
19
+ "float16",
20
+ ])
21
+ .unwrap();
22
+
23
+ expect![[r#"
24
+ Cli {
25
+ command: Serve(
26
+ ServeArgs {
27
+ headless: false,
28
+ host: "127.0.0.1",
29
+ port: 8000,
30
+ uds: None,
31
+ runtime: SharedRuntimeArgs {
32
+ model: "Qwen/Qwen3-0.6B",
33
+ engine_ready_timeout_secs: 600,
34
+ tool_call_parser: Auto,
35
+ reasoning_parser: Auto,
36
+ renderer: Auto,
37
+ language_model_only: false,
38
+ max_model_len: Some(
39
+ 512,
40
+ ),
41
+ max_logprobs: None,
42
+ grpc_port: None,
43
+ shutdown_timeout: 0,
44
+ http_timeout_keep_alive: None,
45
+ chat_template: None,
46
+ default_chat_template_kwargs: None,
47
+ chat_template_content_format: Auto,
48
+ enable_log_requests: false,
49
+ enable_prompt_tokens_details: false,
50
+ enable_request_id_headers: false,
51
+ disable_log_stats: false,
52
+ served_model_name: [],
53
+ allowed_origins: JsonStringList(
54
+ [
55
+ "*",
56
+ ],
57
+ ),
58
+ allowed_methods: JsonStringList(
59
+ [
60
+ "*",
61
+ ],
62
+ ),
63
+ allowed_headers: JsonStringList(
64
+ [
65
+ "*",
66
+ ],
67
+ ),
68
+ allow_credentials: false,
69
+ ssl_keyfile: None,
70
+ ssl_certfile: None,
71
+ ssl_ca_certs: None,
72
+ ssl_cert_reqs: 0,
73
+ ssl_ciphers: None,
74
+ profiler_config: None,
75
+ },
76
+ managed_engine: ManagedEngineArgs {
77
+ python: "../vllm/.venv/bin/python",
78
+ handshake_host: "127.0.0.1",
79
+ handshake_port: None,
80
+ data_parallel_size: 1,
81
+ data_parallel_size_local: None,
82
+ python_args: [
83
+ "--dtype",
84
+ "float16",
85
+ ],
86
+ },
87
+ },
88
+ ),
89
+ }
90
+ "#]]
91
+ .assert_debug_eq(&cli);
92
+ }
93
+
94
+ #[test]
95
+ fn serve_args_auto_forward_python_flags_without_separator() {
96
+ let cli = Cli::try_parse_from([
97
+ "vllm-rs",
98
+ "serve",
99
+ "Qwen/Qwen3-0.6B",
100
+ "--python",
101
+ "python3",
102
+ "--quantization",
103
+ "awq",
104
+ ])
105
+ .unwrap();
106
+
107
+ let Command::Serve(args) = cli.command else {
108
+ panic!("expected serve args");
109
+ };
110
+ expect![[r#"
111
+ [
112
+ "--quantization",
113
+ "awq",
114
+ ]
115
+ "#]]
116
+ .assert_debug_eq(&args.managed_engine.python_args);
117
+ }
118
+
119
+ #[test]
120
+ fn serve_args_auto_forward_enable_lora_to_python() {
121
+ let cli =
122
+ Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--enable-lora"]).unwrap();
123
+
124
+ let Command::Serve(args) = cli.command else {
125
+ panic!("expected serve args");
126
+ };
127
+ expect![[r#"
128
+ [
129
+ "--enable-lora",
130
+ ]
131
+ "#]]
132
+ .assert_debug_eq(&args.managed_engine.python_args);
133
+ }
134
+
135
+ #[test]
136
+ fn serve_args_forward_shutdown_timeout_to_managed_engine() {
137
+ let cli = Cli::try_parse_from([
138
+ "vllm-rs",
139
+ "serve",
140
+ "Qwen/Qwen3-0.6B",
141
+ "--shutdown-timeout",
142
+ "60",
143
+ ])
144
+ .unwrap();
145
+
146
+ let Command::Serve(args) = cli.command else {
147
+ panic!("expected serve args");
148
+ };
149
+ assert_eq!(args.runtime.shutdown_timeout, 60);
150
+
151
+ let config = args.to_managed_engine_config(5555);
152
+ expect![[r#"
153
+ [
154
+ "--reasoning-parser",
155
+ "qwen3",
156
+ "--shutdown-timeout",
157
+ "60",
158
+ ]
159
+ "#]]
160
+ .assert_debug_eq(&config.python_args);
161
+ }
162
+
163
+ #[test]
164
+ fn serve_args_forward_disable_log_stats_to_managed_engine() {
165
+ let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--disable-log-stats"])
166
+ .unwrap();
167
+
168
+ let Command::Serve(args) = cli.command else {
169
+ panic!("expected serve args");
170
+ };
171
+ assert!(args.runtime.disable_log_stats);
172
+
173
+ let config = args.to_managed_engine_config(5555);
174
+ expect![[r#"
175
+ [
176
+ "--reasoning-parser",
177
+ "qwen3",
178
+ "--disable-log-stats",
179
+ ]
180
+ "#]]
181
+ .assert_debug_eq(&config.python_args);
182
+ }
183
+
184
+ #[test]
185
+ fn serve_args_forward_profiler_config_to_managed_engine() {
186
+ let cli = Cli::try_parse_from([
187
+ "vllm-rs",
188
+ "serve",
189
+ "Qwen/Qwen3-0.6B",
190
+ "--profiler-config",
191
+ r#"{"profiler":"torch","torch_profiler_dir":"/tmp/profile"}"#,
192
+ ])
193
+ .unwrap();
194
+
195
+ let Command::Serve(args) = cli.command else {
196
+ panic!("expected serve args");
197
+ };
198
+ assert_eq!(args.runtime.profiler().as_deref(), Some("torch"));
199
+
200
+ let config = args.to_managed_engine_config(5555);
201
+ let profiler_flag_index = config
202
+ .python_args
203
+ .iter()
204
+ .position(|arg| arg == "--profiler-config")
205
+ .expect("profiler config flag");
206
+ let profiler_config: serde_json::Value =
207
+ serde_json::from_str(&config.python_args[profiler_flag_index + 1])
208
+ .expect("profiler config json");
209
+ assert_eq!(
210
+ profiler_config,
211
+ serde_json::json!({
212
+ "profiler": "torch",
213
+ "torch_profiler_dir": "/tmp/profile",
214
+ })
215
+ );
216
+ }
217
+
218
+ #[test]
219
+ fn serve_args_forward_max_logprobs_to_frontend_and_managed_engine() {
220
+ let cli = Cli::try_parse_from([
221
+ "vllm-rs",
222
+ "serve",
223
+ "Qwen/Qwen3-0.6B",
224
+ "--max-logprobs",
225
+ "-1",
226
+ ])
227
+ .unwrap();
228
+
229
+ let Command::Serve(args) = cli.command else {
230
+ panic!("expected serve args");
231
+ };
232
+ assert_eq!(args.runtime.max_logprobs, Some(-1));
233
+
234
+ let frontend_config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
235
+ assert_eq!(frontend_config.max_logprobs, Some(-1));
236
+
237
+ let engine_config = args.to_managed_engine_config(5555);
238
+ expect![[r#"
239
+ [
240
+ "--max-logprobs",
241
+ "-1",
242
+ "--reasoning-parser",
243
+ "qwen3",
244
+ ]
245
+ "#]]
246
+ .assert_debug_eq(&engine_config.python_args);
247
+ }
248
+
249
+ #[test]
250
+ fn serve_args_resolve_auto_reasoning_parser_for_managed_engine() {
251
+ let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B"]).unwrap();
252
+
253
+ let Command::Serve(args) = cli.command else {
254
+ panic!("expected serve args");
255
+ };
256
+ assert_eq!(args.runtime.reasoning_parser, ParserSelection::Auto);
257
+
258
+ let config = args.to_managed_engine_config(5555);
259
+ expect![[r#"
260
+ [
261
+ "--reasoning-parser",
262
+ "qwen3",
263
+ ]
264
+ "#]]
265
+ .assert_debug_eq(&config.python_args);
266
+ }
267
+
268
+ #[test]
269
+ fn serve_args_forward_explicit_reasoning_parser_to_managed_engine() {
270
+ let cli = Cli::try_parse_from([
271
+ "vllm-rs",
272
+ "serve",
273
+ "Unknown/Model",
274
+ "--reasoning-parser",
275
+ "deepseek_r1",
276
+ ])
277
+ .unwrap();
278
+
279
+ let Command::Serve(args) = cli.command else {
280
+ panic!("expected serve args");
281
+ };
282
+
283
+ let config = args.to_managed_engine_config(5555);
284
+ expect![[r#"
285
+ [
286
+ "--reasoning-parser",
287
+ "deepseek_r1",
288
+ ]
289
+ "#]]
290
+ .assert_debug_eq(&config.python_args);
291
+ }
292
+
293
+ #[test]
294
+ fn serve_args_do_not_forward_disabled_reasoning_parser_to_managed_engine() {
295
+ let cli = Cli::try_parse_from([
296
+ "vllm-rs",
297
+ "serve",
298
+ "Qwen/Qwen3-0.6B",
299
+ "--reasoning-parser",
300
+ "none",
301
+ ])
302
+ .unwrap();
303
+
304
+ let Command::Serve(args) = cli.command else {
305
+ panic!("expected serve args");
306
+ };
307
+
308
+ let config = args.to_managed_engine_config(5555);
309
+ assert!(config.python_args.is_empty());
310
+ }
311
+
312
+ #[test]
313
+ fn serve_args_forward_reasoning_parser_even_with_passthrough_reasoning_parser() {
314
+ let cli = Cli::try_parse_from([
315
+ "vllm-rs",
316
+ "serve",
317
+ "Qwen/Qwen3-0.6B",
318
+ "--",
319
+ "--reasoning-parser",
320
+ "deepseek_r1",
321
+ ])
322
+ .unwrap();
323
+
324
+ let Command::Serve(args) = cli.command else {
325
+ panic!("expected serve args");
326
+ };
327
+
328
+ let config = args.to_managed_engine_config(5555);
329
+ expect![[r#"
330
+ [
331
+ "--reasoning-parser",
332
+ "deepseek_r1",
333
+ "--reasoning-parser",
334
+ "qwen3",
335
+ ]
336
+ "#]]
337
+ .assert_debug_eq(&config.python_args);
338
+ }
339
+
340
+ #[test]
341
+ fn serve_args_auto_forward_python_multi_char_alias_without_separator() {
342
+ let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "-tp", "2"]).unwrap();
343
+
344
+ let Command::Serve(args) = cli.command else {
345
+ panic!("expected serve args");
346
+ };
347
+ expect![[r#"
348
+ [
349
+ "--tensor-parallel-size",
350
+ "2",
351
+ ]
352
+ "#]]
353
+ .assert_debug_eq(&args.managed_engine.python_args);
354
+ }
355
+
356
+ #[test]
357
+ fn serve_args_accept_explicit_deepseek_v32_renderer() {
358
+ let cli = Cli::try_parse_from([
359
+ "vllm-rs",
360
+ "serve",
361
+ "Qwen/Qwen3-0.6B",
362
+ "--tokenizer-mode",
363
+ "deepseek_v32",
364
+ ])
365
+ .unwrap();
366
+
367
+ let Command::Serve(args) = cli.command else {
368
+ panic!("expected serve args");
369
+ };
370
+ assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32);
371
+ }
372
+
373
+ #[test]
374
+ fn serve_passes_enable_request_id_headers_into_config() {
375
+ let cli = Cli::try_parse_from([
376
+ "vllm-rs",
377
+ "serve",
378
+ "Qwen/Qwen3-0.6B",
379
+ "--enable-request-id-headers",
380
+ ])
381
+ .unwrap();
382
+
383
+ let Command::Serve(args) = cli.command else {
384
+ panic!("expected serve args");
385
+ };
386
+ let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
387
+ assert!(config.api_server_options.enable_request_id_headers);
388
+ }
389
+
390
+ #[test]
391
+ fn serve_passes_enable_prompt_tokens_details_into_config() {
392
+ let cli = Cli::try_parse_from([
393
+ "vllm-rs",
394
+ "serve",
395
+ "Qwen/Qwen3-0.6B",
396
+ "--enable-prompt-tokens-details",
397
+ ])
398
+ .unwrap();
399
+
400
+ let Command::Serve(args) = cli.command else {
401
+ panic!("expected serve args");
402
+ };
403
+ let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
404
+ assert!(config.api_server_options.enable_prompt_tokens_details);
405
+ }
406
+
407
+ #[test]
408
+ fn serve_passes_tls_into_config() {
409
+ let cli = Cli::try_parse_from([
410
+ "vllm-rs",
411
+ "serve",
412
+ "Qwen/Qwen3-0.6B",
413
+ "--ssl-certfile",
414
+ "/tmp/cert.pem",
415
+ "--ssl-keyfile",
416
+ "/tmp/key.pem",
417
+ "--ssl-ca-certs",
418
+ "/tmp/ca.pem",
419
+ "--ssl-cert-reqs",
420
+ "2",
421
+ ])
422
+ .unwrap();
423
+
424
+ let Command::Serve(args) = cli.command else {
425
+ panic!("expected serve args");
426
+ };
427
+ let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
428
+ let tls = config.tls.expect("tls configured");
429
+ assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem"));
430
+ assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem"));
431
+ assert_eq!(tls.ca_certs.as_deref(), Some("/tmp/ca.pem"));
432
+ assert_eq!(tls.cert_reqs, 2);
433
+ }
434
+
435
+ #[test]
436
+ fn serve_without_ssl_flags_has_no_tls() {
437
+ let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B"]).unwrap();
438
+
439
+ let Command::Serve(args) = cli.command else {
440
+ panic!("expected serve args");
441
+ };
442
+ let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
443
+ assert!(config.tls.is_none());
444
+ }
445
+
446
+ #[test]
447
+ fn serve_ssl_keyfile_without_certfile_fails_validation() {
448
+ let cli = Cli::try_parse_from([
449
+ "vllm-rs",
450
+ "serve",
451
+ "Qwen/Qwen3-0.6B",
452
+ "--ssl-keyfile",
453
+ "/tmp/key.pem",
454
+ ])
455
+ .unwrap();
456
+
457
+ let Command::Serve(args) = cli.command else {
458
+ panic!("expected serve args");
459
+ };
460
+ let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
461
+ // TLS is requested (a key was given) but there is no certificate, so
462
+ // validation fails loud rather than silently serving plaintext.
463
+ assert_eq!(config.tls.as_ref().expect("tls requested").cert_file, None);
464
+ let err = config.validate().unwrap_err().to_string();
465
+ assert!(err.contains("--ssl-certfile is required"), "{err}");
466
+ }
467
+
468
+ #[test]
469
+ fn serve_mtls_without_ca_certs_fails_validation() {
470
+ let cli = Cli::try_parse_from([
471
+ "vllm-rs",
472
+ "serve",
473
+ "Qwen/Qwen3-0.6B",
474
+ "--ssl-certfile",
475
+ "/tmp/cert.pem",
476
+ "--ssl-cert-reqs",
477
+ "2",
478
+ ])
479
+ .unwrap();
480
+
481
+ let Command::Serve(args) = cli.command else {
482
+ panic!("expected serve args");
483
+ };
484
+ let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
485
+ // Client-cert verification without a CA bundle has nothing to verify
486
+ // against, so it fails loud at startup.
487
+ let err = config.validate().unwrap_err().to_string();
488
+ assert!(err.contains("--ssl-ca-certs is required"), "{err}");
489
+ }
490
+
491
+ #[test]
492
+ fn frontend_args_json_passes_tls_into_config() {
493
+ let cli = Cli::try_parse_from([
494
+ "vllm-rs",
495
+ "frontend",
496
+ "--listen-fd",
497
+ "3",
498
+ "--input-address",
499
+ "ipc:///tmp/input.sock",
500
+ "--output-address",
501
+ "ipc:///tmp/output.sock",
502
+ "--args-json",
503
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_certfile":"/tmp/cert.pem","ssl_keyfile":"/tmp/key.pem"}"#,
504
+ ])
505
+ .unwrap();
506
+
507
+ let Command::Frontend(args) = cli.command else {
508
+ panic!("expected frontend args");
509
+ };
510
+ let config = args.into_config();
511
+ let tls = config.tls.expect("tls configured");
512
+ assert_eq!(tls.cert_file.as_deref(), Some("/tmp/cert.pem"));
513
+ assert_eq!(tls.key_file.as_deref(), Some("/tmp/key.pem"));
514
+ }
515
+
516
+ #[test]
517
+ fn frontend_args_json_rejects_out_of_range_cert_reqs() {
518
+ let cli = Cli::try_parse_from([
519
+ "vllm-rs",
520
+ "frontend",
521
+ "--listen-fd",
522
+ "3",
523
+ "--input-address",
524
+ "ipc:///tmp/input.sock",
525
+ "--output-address",
526
+ "ipc:///tmp/output.sock",
527
+ "--args-json",
528
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","ssl_certfile":"/tmp/cert.pem","ssl_cert_reqs":5}"#,
529
+ ])
530
+ .unwrap();
531
+
532
+ let Command::Frontend(args) = cli.command else {
533
+ panic!("expected frontend args");
534
+ };
535
+ // The JSON path bypasses clap's range check, so validate() is the only guard.
536
+ let config = args.into_config();
537
+ let err = config.validate().unwrap_err().to_string();
538
+ assert!(err.contains("--ssl-cert-reqs"), "{err}");
539
+ }
540
+
541
+ #[test]
542
+ fn frontend_args_json_passes_enable_request_id_headers_into_config() {
543
+ let cli = Cli::try_parse_from([
544
+ "vllm-rs",
545
+ "frontend",
546
+ "--listen-fd",
547
+ "3",
548
+ "--input-address",
549
+ "ipc:///tmp/input.sock",
550
+ "--output-address",
551
+ "ipc:///tmp/output.sock",
552
+ "--args-json",
553
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","enable_request_id_headers":true}"#,
554
+ ])
555
+ .unwrap();
556
+
557
+ let Command::Frontend(args) = cli.command else {
558
+ panic!("expected frontend args");
559
+ };
560
+ let config = args.into_config();
561
+ assert!(config.api_server_options.enable_request_id_headers);
562
+ }
563
+
564
+ #[test]
565
+ fn serve_passes_api_keys_into_config() {
566
+ let cli = Cli::try_parse_from([
567
+ "vllm-rs",
568
+ "serve",
569
+ "Qwen/Qwen3-0.6B",
570
+ "--api-key",
571
+ "secret-a",
572
+ "--api-key",
573
+ "secret-b",
574
+ ])
575
+ .unwrap();
576
+
577
+ let Command::Serve(args) = cli.command else {
578
+ panic!("expected serve args");
579
+ };
580
+ let config = args.to_frontend_config("tcp://127.0.0.1:62100".to_string());
581
+ assert_eq!(config.api_keys, vec!["secret-a", "secret-b"]);
582
+ let debug = format!("{config:#?}");
583
+ assert!(debug.contains("api_keys: [<redacted>; 2]"));
584
+ assert!(!debug.contains("secret-a"));
585
+ assert!(!debug.contains("secret-b"));
586
+ }
587
+
588
+ #[test]
589
+ fn frontend_args_json_accepts_api_key_string() {
590
+ let cli = Cli::try_parse_from([
591
+ "vllm-rs",
592
+ "frontend",
593
+ "--listen-fd",
594
+ "3",
595
+ "--input-address",
596
+ "ipc:///tmp/input.sock",
597
+ "--output-address",
598
+ "ipc:///tmp/output.sock",
599
+ "--args-json",
600
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","api_key":"secret"}"#,
601
+ ])
602
+ .unwrap();
603
+
604
+ let Command::Frontend(args) = cli.command else {
605
+ panic!("expected frontend args");
606
+ };
607
+ let config = args.into_config();
608
+ assert_eq!(config.api_keys, vec!["secret"]);
609
+ }
610
+
611
+ #[test]
612
+ fn frontend_args_json_accepts_api_key_list() {
613
+ let cli = Cli::try_parse_from([
614
+ "vllm-rs",
615
+ "frontend",
616
+ "--listen-fd",
617
+ "3",
618
+ "--input-address",
619
+ "ipc:///tmp/input.sock",
620
+ "--output-address",
621
+ "ipc:///tmp/output.sock",
622
+ "--args-json",
623
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","api_key":["secret-a","secret-b"]}"#,
624
+ ])
625
+ .unwrap();
626
+
627
+ let Command::Frontend(args) = cli.command else {
628
+ panic!("expected frontend args");
629
+ };
630
+ let config = args.into_config();
631
+ assert_eq!(config.api_keys, vec!["secret-a", "secret-b"]);
632
+ }
633
+
634
+ #[test]
635
+ fn serve_args_reject_unknown_renderer_value() {
636
+ let error = Cli::try_parse_from([
637
+ "vllm-rs",
638
+ "serve",
639
+ "Qwen/Qwen3-0.6B",
640
+ "--tokenizer-mode",
641
+ "definitely_missing",
642
+ ])
643
+ .unwrap_err();
644
+
645
+ expect![[r#"
646
+ error: invalid value 'definitely_missing' for '--tokenizer-mode <RENDERER>': unknown renderer `definitely_missing` (expected one of: auto, hf, deepseek_v32, deepseek_v4, harmony)
647
+
648
+ For more information, try '--help'.
649
+ "#]]
650
+ .assert_eq(&error.to_string());
651
+ }
652
+
653
+ #[test]
654
+ fn serve_args_reject_unsupported_flag_arg() {
655
+ let error = Cli::try_parse_from([
656
+ "vllm-rs",
657
+ "serve",
658
+ "Qwen/Qwen3-0.6B",
659
+ "--root-path",
660
+ "/prefix",
661
+ ])
662
+ .unwrap_err();
663
+
664
+ expect![[r#"
665
+ error: invalid value '/prefix' for '--root-path <ROOT_PATH>': argument is not implemented in Rust frontend yet
666
+
667
+ Remove this unsupported argument to continue.
668
+
669
+ Alternatively, if you intend to pass it only to the Python engine, put it after `--` (e.g., `-- <arg>`).
670
+ This may lead to unexpected behavior as the Rust frontend will completely ignore that argument.
671
+
672
+ For more information, try '--help'.
673
+ "#]].assert_eq(&error.to_string());
674
+ }
675
+
676
+ #[test]
677
+ fn serve_args_reject_unsupported_no_flag_alias() {
678
+ let error = Cli::try_parse_from([
679
+ "vllm-rs",
680
+ "serve",
681
+ "Qwen/Qwen3-0.6B",
682
+ "--no-enable-log-deltas",
683
+ ])
684
+ .unwrap_err();
685
+
686
+ expect![[r#"
687
+ error: invalid value 'true' for '--enable-log-deltas [<ENABLE_LOG_DELTAS>]': argument is not implemented in Rust frontend yet
688
+
689
+ Remove this unsupported argument to continue.
690
+
691
+ Alternatively, if you intend to pass it only to the Python engine, put it after `--` (e.g., `-- <arg>`).
692
+ This may lead to unexpected behavior as the Rust frontend will completely ignore that argument.
693
+
694
+ For more information, try '--help'.
695
+ "#]]
696
+ .assert_eq(&error.to_string());
697
+ }
698
+
699
+ #[test]
700
+ fn frontend_args_accept_json() {
701
+ let cli = Cli::try_parse_from([
702
+ "vllm-rs",
703
+ "frontend",
704
+ "--listen-fd",
705
+ "3",
706
+ "--input-address",
707
+ "ipc:///tmp/input.sock",
708
+ "--output-address",
709
+ "ipc:///tmp/output.sock",
710
+ "--coordinator-address",
711
+ "tcp://127.0.0.1:7000",
712
+ "--args-json",
713
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_count":2}"#,
714
+ ])
715
+ .unwrap();
716
+
717
+ expect![[r#"
718
+ Cli {
719
+ command: Frontend(
720
+ FrontendArgs {
721
+ listen_fd: 3,
722
+ input_address: "ipc:///tmp/input.sock",
723
+ output_address: "ipc:///tmp/output.sock",
724
+ coordinator_address: Some(
725
+ "tcp://127.0.0.1:7000",
726
+ ),
727
+ engine_start_index: 0,
728
+ engine_count: 1,
729
+ runtime: SharedRuntimeArgs {
730
+ model: "Qwen/Qwen3-0.6B",
731
+ engine_ready_timeout_secs: 600,
732
+ tool_call_parser: None,
733
+ reasoning_parser: None,
734
+ renderer: Auto,
735
+ language_model_only: false,
736
+ max_model_len: None,
737
+ max_logprobs: None,
738
+ grpc_port: None,
739
+ shutdown_timeout: 0,
740
+ http_timeout_keep_alive: None,
741
+ chat_template: None,
742
+ default_chat_template_kwargs: None,
743
+ chat_template_content_format: Auto,
744
+ enable_log_requests: false,
745
+ enable_prompt_tokens_details: false,
746
+ enable_request_id_headers: false,
747
+ disable_log_stats: false,
748
+ served_model_name: [],
749
+ allowed_origins: JsonStringList(
750
+ [
751
+ "*",
752
+ ],
753
+ ),
754
+ allowed_methods: JsonStringList(
755
+ [
756
+ "*",
757
+ ],
758
+ ),
759
+ allowed_headers: JsonStringList(
760
+ [
761
+ "*",
762
+ ],
763
+ ),
764
+ allow_credentials: false,
765
+ ssl_keyfile: None,
766
+ ssl_certfile: None,
767
+ ssl_ca_certs: None,
768
+ ssl_cert_reqs: 0,
769
+ ssl_ciphers: None,
770
+ profiler_config: None,
771
+ },
772
+ },
773
+ ),
774
+ }
775
+ "#]]
776
+ .assert_debug_eq(&cli);
777
+ }
778
+
779
+ #[test]
780
+ fn frontend_args_json_applies_defaults() {
781
+ let cli = Cli::try_parse_from([
782
+ "vllm-rs",
783
+ "frontend",
784
+ "--listen-fd",
785
+ "3",
786
+ "--input-address",
787
+ "ipc:///tmp/input.sock",
788
+ "--output-address",
789
+ "ipc:///tmp/output.sock",
790
+ "--args-json",
791
+ r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
792
+ ])
793
+ .unwrap();
794
+
795
+ let Command::Frontend(args) = cli.command else {
796
+ panic!("expected frontend args");
797
+ };
798
+ assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
799
+ assert_eq!(args.runtime.engine_ready_timeout_secs, 600);
800
+ assert_eq!(args.runtime.tool_call_parser, ParserSelection::None);
801
+ assert_eq!(args.runtime.reasoning_parser, ParserSelection::None);
802
+ assert_eq!(args.runtime.renderer, RendererSelection::Auto);
803
+ assert_eq!(args.runtime.max_model_len, None);
804
+ assert_eq!(args.runtime.max_logprobs, None);
805
+ assert_eq!(args.runtime.shutdown_timeout, 0);
806
+ }
807
+
808
+ #[test]
809
+ fn frontend_args_json_accepts_supported_non_default_fields() {
810
+ let cli = Cli::try_parse_from([
811
+ "vllm-rs",
812
+ "frontend",
813
+ "--listen-fd",
814
+ "3",
815
+ "--input-address",
816
+ "ipc:///tmp/input.sock",
817
+ "--output-address",
818
+ "ipc:///tmp/output.sock",
819
+ "--args-json",
820
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","engine_ready_timeout_secs":42,"tool_call_parser":"hermes","reasoning_parser":"qwen3_thinking","tokenizer_mode":"deepseek_v32","language_model_only":true,"max_model_len":8192,"max_logprobs":-1,"shutdown_timeout":3}"#,
821
+ ])
822
+ .unwrap();
823
+
824
+ let Command::Frontend(args) = cli.command else {
825
+ panic!("expected frontend args");
826
+ };
827
+ assert_eq!(args.runtime.engine_ready_timeout_secs, 42);
828
+ assert_eq!(
829
+ args.runtime.tool_call_parser,
830
+ ParserSelection::Explicit("hermes".to_string())
831
+ );
832
+ assert_eq!(
833
+ args.runtime.reasoning_parser,
834
+ ParserSelection::Explicit("qwen3_thinking".to_string())
835
+ );
836
+ assert_eq!(args.runtime.renderer, RendererSelection::DeepSeekV32);
837
+ assert!(args.runtime.language_model_only);
838
+ assert_eq!(args.runtime.max_model_len, Some(8192));
839
+ assert_eq!(args.runtime.max_logprobs, Some(-1));
840
+ assert_eq!(args.runtime.shutdown_timeout, 3);
841
+ }
842
+
843
+ #[test]
844
+ fn serve_args_accept_none_reasoning_parser() {
845
+ let cli = Cli::try_parse_from([
846
+ "vllm-rs",
847
+ "serve",
848
+ "Qwen/Qwen3-0.6B",
849
+ "--reasoning-parser",
850
+ "none",
851
+ ])
852
+ .unwrap();
853
+
854
+ let Command::Serve(args) = cli.command else {
855
+ panic!("expected serve args");
856
+ };
857
+ assert_eq!(args.runtime.reasoning_parser, ParserSelection::None);
858
+ assert_eq!(args.runtime.tool_call_parser, ParserSelection::Auto);
859
+ }
860
+
861
+ #[test]
862
+ fn frontend_args_json_ignores_unknown_fields() {
863
+ let cli = Cli::try_parse_from([
864
+ "vllm-rs",
865
+ "frontend",
866
+ "--listen-fd",
867
+ "3",
868
+ "--input-address",
869
+ "ipc:///tmp/input.sock",
870
+ "--output-address",
871
+ "ipc:///tmp/output.sock",
872
+ "--args-json",
873
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","uds":"/tmp/vllm.sock","nested_unknown":{"x":1}}"#,
874
+ ])
875
+ .unwrap();
876
+
877
+ let Command::Frontend(args) = cli.command else {
878
+ panic!("expected frontend args");
879
+ };
880
+ assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
881
+ }
882
+
883
+ #[test]
884
+ fn frontend_args_json_sets_prompt_tokens_details_flag() {
885
+ let cli = Cli::try_parse_from([
886
+ "vllm-rs",
887
+ "frontend",
888
+ "--listen-fd",
889
+ "3",
890
+ "--input-address",
891
+ "ipc:///tmp/input.sock",
892
+ "--output-address",
893
+ "ipc:///tmp/output.sock",
894
+ "--args-json",
895
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","api_server_count":2,"enable_prompt_tokens_details":true}"#,
896
+ ])
897
+ .unwrap();
898
+
899
+ let Command::Frontend(args) = cli.command else {
900
+ panic!("expected frontend args");
901
+ };
902
+ assert_eq!(args.runtime.model, "Qwen/Qwen3-0.6B");
903
+ assert!(args.runtime.enable_prompt_tokens_details);
904
+ }
905
+
906
+ #[test]
907
+ fn serve_args_parse_cors_flags() {
908
+ let cli = Cli::try_parse_from([
909
+ "vllm-rs",
910
+ "serve",
911
+ "Qwen/Qwen3-0.6B",
912
+ "--allowed-origins",
913
+ r#"["http://a.com","http://b.com"]"#,
914
+ "--allowed-methods",
915
+ r#"["GET","POST"]"#,
916
+ "--allow-credentials",
917
+ ])
918
+ .unwrap();
919
+
920
+ let Command::Serve(serve) = cli.command else {
921
+ panic!("expected serve args");
922
+ };
923
+ assert_eq!(
924
+ serve.runtime.allowed_origins.0,
925
+ ["http://a.com", "http://b.com"]
926
+ );
927
+ assert_eq!(serve.runtime.allowed_methods.0, ["GET", "POST"]);
928
+ assert!(serve.runtime.allow_credentials);
929
+ // Unspecified lists keep the permissive default.
930
+ assert_eq!(serve.runtime.allowed_headers.0, ["*"]);
931
+ }
932
+
933
+ #[test]
934
+ fn serve_args_cors_defaults_are_permissive() {
935
+ let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B"]).unwrap();
936
+
937
+ let Command::Serve(serve) = cli.command else {
938
+ panic!("expected serve args");
939
+ };
940
+ assert_eq!(serve.runtime.allowed_origins.0, ["*"]);
941
+ assert_eq!(serve.runtime.allowed_methods.0, ["*"]);
942
+ assert_eq!(serve.runtime.allowed_headers.0, ["*"]);
943
+ assert!(!serve.runtime.allow_credentials);
944
+ }
945
+
946
+ #[test]
947
+ fn frontend_args_json_parses_cors_fields() {
948
+ let cli = Cli::try_parse_from([
949
+ "vllm-rs",
950
+ "frontend",
951
+ "--listen-fd",
952
+ "3",
953
+ "--input-address",
954
+ "ipc:///tmp/input.sock",
955
+ "--output-address",
956
+ "ipc:///tmp/output.sock",
957
+ "--args-json",
958
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","allowed_origins":["http://a.com"],"allow_credentials":true}"#,
959
+ ])
960
+ .unwrap();
961
+
962
+ let Command::Frontend(args) = cli.command else {
963
+ panic!("expected frontend args");
964
+ };
965
+ assert_eq!(args.runtime.allowed_origins.0, ["http://a.com"]);
966
+ assert!(args.runtime.allow_credentials);
967
+ // Unspecified lists fall back to the permissive default via serde.
968
+ assert_eq!(args.runtime.allowed_methods.0, ["*"]);
969
+ }
970
+
971
+ #[test]
972
+ fn frontend_args_json_rejects_unsupported_fields() {
973
+ let error = Cli::try_parse_from([
974
+ "vllm-rs",
975
+ "frontend",
976
+ "--listen-fd",
977
+ "3",
978
+ "--input-address",
979
+ "ipc:///tmp/input.sock",
980
+ "--output-address",
981
+ "ipc:///tmp/output.sock",
982
+ "--args-json",
983
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","root_path":"/prefix"}"#,
984
+ ])
985
+ .unwrap_err();
986
+
987
+ expect![[r#"
988
+ error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","root_path":"/prefix"}' for '--args-json <JSON>':
989
+ The following arguments are not implemented in Rust frontend yet:
990
+ - root_path
991
+
992
+ Remove these arguments to continue.
993
+
994
+ For more information, try '--help'.
995
+ "#]].assert_eq(&error.to_string());
996
+ }
997
+
998
+ #[test]
999
+ fn frontend_args_json_aggregates_multiple_unsupported_fields() {
1000
+ let error = Cli::try_parse_from([
1001
+ "vllm-rs",
1002
+ "frontend",
1003
+ "--listen-fd",
1004
+ "3",
1005
+ "--input-address",
1006
+ "ipc:///tmp/input.sock",
1007
+ "--output-address",
1008
+ "ipc:///tmp/output.sock",
1009
+ "--args-json",
1010
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","root_path":"/prefix"}"#,
1011
+ ])
1012
+ .unwrap_err();
1013
+
1014
+ let actual = error.to_string().replace(": \n", ":\n");
1015
+ expect![[r#"
1016
+ error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B","response_role":"assistant","root_path":"/prefix"}' for '--args-json <JSON>':
1017
+ The following arguments are not implemented in Rust frontend yet:
1018
+ - response_role
1019
+ - root_path
1020
+
1021
+ Remove these arguments to continue.
1022
+
1023
+ For more information, try '--help'.
1024
+ "#]].assert_eq(&actual);
1025
+ }
1026
+
1027
+ #[test]
1028
+ fn frontend_args_json_rejects_malformed_json() {
1029
+ let error = Cli::try_parse_from([
1030
+ "vllm-rs",
1031
+ "frontend",
1032
+ "--listen-fd",
1033
+ "3",
1034
+ "--input-address",
1035
+ "ipc:///tmp/input.sock",
1036
+ "--output-address",
1037
+ "ipc:///tmp/output.sock",
1038
+ "--args-json",
1039
+ r#"{"model_tag":"Qwen/Qwen3-0.6B""#,
1040
+ ])
1041
+ .unwrap_err();
1042
+
1043
+ expect![[r#"
1044
+ error: invalid value '{"model_tag":"Qwen/Qwen3-0.6B"' for '--args-json <JSON>': invalid JSON arguments: EOF while parsing an object at line 1 column 30
1045
+
1046
+ For more information, try '--help'.
1047
+ "#]].assert_eq(&error.to_string());
1048
+ }
1049
+
1050
+ #[test]
1051
+ fn serve_args_reject_flags_before_model() {
1052
+ let error = Cli::try_parse_from(["vllm-rs", "serve", "--python", "python3", "Qwen/Qwen3-0.6B"])
1053
+ .unwrap_err();
1054
+
1055
+ expect![[r#"
1056
+ error: the model must appear immediately after the command
1057
+
1058
+ Usage: vllm-rs serve <MODEL> [OPTIONS] [-- <PYTHON_ARGS>...]
1059
+
1060
+ For more information, try '--help'.
1061
+ "#]]
1062
+ .assert_eq(&error.to_string());
1063
+ }
1064
+
1065
+ #[test]
1066
+ fn serve_args_accept_headless_mode() {
1067
+ let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--headless"]).unwrap();
1068
+
1069
+ let Command::Serve(args) = cli.command else {
1070
+ panic!("expected serve args");
1071
+ };
1072
+ assert!(args.headless);
1073
+ }
1074
+
1075
+ #[test]
1076
+ fn serve_args_keep_python_passthrough_flags_after_separator() {
1077
+ let cli = Cli::try_parse_from([
1078
+ "vllm-rs",
1079
+ "serve",
1080
+ "Qwen/Qwen3-0.6B",
1081
+ "--python",
1082
+ "python3",
1083
+ "--",
1084
+ "--tensor-parallel-size",
1085
+ "2",
1086
+ "--dtype",
1087
+ "float16",
1088
+ ])
1089
+ .unwrap();
1090
+
1091
+ let Command::Serve(args) = cli.command else {
1092
+ panic!("expected serve args");
1093
+ };
1094
+ expect![[r#"
1095
+ [
1096
+ "--tensor-parallel-size",
1097
+ "2",
1098
+ "--dtype",
1099
+ "float16",
1100
+ ]
1101
+ "#]]
1102
+ .assert_debug_eq(&args.managed_engine.python_args);
1103
+ }
1104
+
1105
+ #[test]
1106
+ fn serve_args_keep_python_multi_char_alias_after_separator() {
1107
+ let cli = Cli::try_parse_from([
1108
+ "vllm-rs",
1109
+ "serve",
1110
+ "Qwen/Qwen3-0.6B",
1111
+ "--python",
1112
+ "python3",
1113
+ "--",
1114
+ "-tp",
1115
+ "2",
1116
+ "--dtype",
1117
+ "float16",
1118
+ ])
1119
+ .unwrap();
1120
+
1121
+ let Command::Serve(args) = cli.command else {
1122
+ panic!("expected serve args");
1123
+ };
1124
+ expect![[r#"
1125
+ [
1126
+ "-tp",
1127
+ "2",
1128
+ "--dtype",
1129
+ "float16",
1130
+ ]
1131
+ "#]]
1132
+ .assert_debug_eq(&args.managed_engine.python_args);
1133
+ }
1134
+
1135
+ #[test]
1136
+ fn serve_args_keep_frontend_arg_after_separator() {
1137
+ let cli = Cli::try_parse_from([
1138
+ "vllm-rs",
1139
+ "serve",
1140
+ "Qwen/Qwen3-0.6B",
1141
+ "--",
1142
+ "--uds",
1143
+ "/tmp/vllm.sock",
1144
+ ])
1145
+ .unwrap();
1146
+
1147
+ let Command::Serve(args) = cli.command else {
1148
+ panic!("expected serve args");
1149
+ };
1150
+ expect![[r#"
1151
+ [
1152
+ "--uds",
1153
+ "/tmp/vllm.sock",
1154
+ ]
1155
+ "#]]
1156
+ .assert_debug_eq(&args.managed_engine.python_args);
1157
+ }
1158
+
1159
+ #[test]
1160
+ fn serve_args_keep_python_multi_char_engine_aliases_after_separator() {
1161
+ let cli = Cli::try_parse_from([
1162
+ "vllm-rs",
1163
+ "serve",
1164
+ "Qwen/Qwen3-0.6B",
1165
+ "--",
1166
+ "-dpr",
1167
+ "1",
1168
+ "-dpl",
1169
+ "2",
1170
+ ])
1171
+ .unwrap();
1172
+
1173
+ let Command::Serve(args) = cli.command else {
1174
+ panic!("expected serve args");
1175
+ };
1176
+ expect![[r#"
1177
+ [
1178
+ "-dpr",
1179
+ "1",
1180
+ "-dpl",
1181
+ "2",
1182
+ ]
1183
+ "#]]
1184
+ .assert_debug_eq(&args.managed_engine.python_args);
1185
+ }
1186
+
1187
+ #[test]
1188
+ fn serve_args_auto_forward_unknown_flags_without_separator() {
1189
+ let cli = Cli::try_parse_from(["vllm-rs", "serve", "Qwen/Qwen3-0.6B", "--foo", "bar"]).unwrap();
1190
+
1191
+ let Command::Serve(args) = cli.command else {
1192
+ panic!("expected serve args");
1193
+ };
1194
+ expect![[r#"
1195
+ [
1196
+ "--foo",
1197
+ "bar",
1198
+ ]
1199
+ "#]]
1200
+ .assert_debug_eq(&args.managed_engine.python_args);
1201
+ }
1202
+
1203
+ #[test]
1204
+ fn serve_args_auto_forward_negative_value_without_separator() {
1205
+ let cli = Cli::try_parse_from([
1206
+ "vllm-rs",
1207
+ "serve",
1208
+ "Qwen/Qwen3-0.6B",
1209
+ "--num-gpu-blocks-override",
1210
+ "-1",
1211
+ ])
1212
+ .unwrap();
1213
+
1214
+ let Command::Serve(args) = cli.command else {
1215
+ panic!("expected serve args");
1216
+ };
1217
+ expect![[r#"
1218
+ [
1219
+ "--num-gpu-blocks-override",
1220
+ "-1",
1221
+ ]
1222
+ "#]]
1223
+ .assert_debug_eq(&args.managed_engine.python_args);
1224
+ }
1225
+
1226
+ #[test]
1227
+ fn serve_args_accept_handshake_aliases() {
1228
+ let cli = Cli::try_parse_from([
1229
+ "vllm-rs",
1230
+ "serve",
1231
+ "Qwen/Qwen3-0.6B",
1232
+ "--python",
1233
+ "python3",
1234
+ "--handshake-host",
1235
+ "10.99.48.128",
1236
+ "--handshake-port",
1237
+ "13345",
1238
+ "--data-parallel-size",
1239
+ "4",
1240
+ ])
1241
+ .unwrap();
1242
+
1243
+ expect![[r#"
1244
+ Cli {
1245
+ command: Serve(
1246
+ ServeArgs {
1247
+ headless: false,
1248
+ host: "127.0.0.1",
1249
+ port: 8000,
1250
+ uds: None,
1251
+ runtime: SharedRuntimeArgs {
1252
+ model: "Qwen/Qwen3-0.6B",
1253
+ engine_ready_timeout_secs: 600,
1254
+ tool_call_parser: Auto,
1255
+ reasoning_parser: Auto,
1256
+ renderer: Auto,
1257
+ language_model_only: false,
1258
+ max_model_len: None,
1259
+ max_logprobs: None,
1260
+ grpc_port: None,
1261
+ shutdown_timeout: 0,
1262
+ http_timeout_keep_alive: None,
1263
+ chat_template: None,
1264
+ default_chat_template_kwargs: None,
1265
+ chat_template_content_format: Auto,
1266
+ enable_log_requests: false,
1267
+ enable_prompt_tokens_details: false,
1268
+ enable_request_id_headers: false,
1269
+ disable_log_stats: false,
1270
+ served_model_name: [],
1271
+ allowed_origins: JsonStringList(
1272
+ [
1273
+ "*",
1274
+ ],
1275
+ ),
1276
+ allowed_methods: JsonStringList(
1277
+ [
1278
+ "*",
1279
+ ],
1280
+ ),
1281
+ allowed_headers: JsonStringList(
1282
+ [
1283
+ "*",
1284
+ ],
1285
+ ),
1286
+ allow_credentials: false,
1287
+ ssl_keyfile: None,
1288
+ ssl_certfile: None,
1289
+ ssl_ca_certs: None,
1290
+ ssl_cert_reqs: 0,
1291
+ ssl_ciphers: None,
1292
+ profiler_config: None,
1293
+ },
1294
+ managed_engine: ManagedEngineArgs {
1295
+ python: "python3",
1296
+ handshake_host: "10.99.48.128",
1297
+ handshake_port: Some(
1298
+ 13345,
1299
+ ),
1300
+ data_parallel_size: 4,
1301
+ data_parallel_size_local: None,
1302
+ python_args: [],
1303
+ },
1304
+ },
1305
+ ),
1306
+ }
1307
+ "#]]
1308
+ .assert_debug_eq(&cli);
1309
+ }
1310
+
1311
+ #[test]
1312
+ fn serve_args_accept_data_parallel_primary_flags() {
1313
+ let cli = Cli::try_parse_from([
1314
+ "vllm-rs",
1315
+ "serve",
1316
+ "Qwen/Qwen3-0.6B",
1317
+ "--data-parallel-address",
1318
+ "10.99.48.128",
1319
+ "--data-parallel-rpc-port",
1320
+ "13345",
1321
+ "--data-parallel-size",
1322
+ "4",
1323
+ ])
1324
+ .unwrap();
1325
+
1326
+ let Command::Serve(args) = cli.command else {
1327
+ panic!("expected serve args");
1328
+ };
1329
+ assert!(!args.headless);
1330
+ assert_eq!(args.managed_engine.handshake_host, "10.99.48.128");
1331
+ assert_eq!(args.managed_engine.handshake_port, Some(13345));
1332
+ assert_eq!(args.managed_engine.data_parallel_size, 4);
1333
+ }
1334
+
1335
+ #[test]
1336
+ fn serve_frontend_config_uses_dp_address_as_advertised_host() {
1337
+ let cli = Cli::try_parse_from([
1338
+ "vllm-rs",
1339
+ "serve",
1340
+ "Qwen/Qwen3-0.6B",
1341
+ "--handshake-host",
1342
+ "10.99.48.128",
1343
+ "--data-parallel-size",
1344
+ "4",
1345
+ ])
1346
+ .unwrap();
1347
+
1348
+ let Command::Serve(args) = cli.command else {
1349
+ panic!("expected serve args");
1350
+ };
1351
+ let config = args.to_frontend_config("tcp://10.99.48.128:29550".to_string());
1352
+
1353
+ let TransportMode::HandshakeOwner {
1354
+ handshake_address,
1355
+ advertised_host,
1356
+ engine_count,
1357
+ ready_timeout,
1358
+ local_input_address,
1359
+ local_output_address,
1360
+ } = &config.transport_mode
1361
+ else {
1362
+ panic!("expected handshake-owned transport");
1363
+ };
1364
+
1365
+ assert_eq!(handshake_address, "tcp://10.99.48.128:29550");
1366
+ assert_eq!(advertised_host, "10.99.48.128");
1367
+ assert_eq!(*engine_count, 4);
1368
+ assert!(
1369
+ local_input_address
1370
+ .as_deref()
1371
+ .is_some_and(|address| address.starts_with("ipc://"))
1372
+ );
1373
+ assert!(
1374
+ local_output_address
1375
+ .as_deref()
1376
+ .is_some_and(|address| address.starts_with("ipc://"))
1377
+ );
1378
+ assert_ne!(local_input_address, local_output_address);
1379
+
1380
+ expect![[r#"
1381
+ Config {
1382
+ transport_mode: HandshakeOwner {
1383
+ handshake_address: "tcp://10.99.48.128:29550",
1384
+ advertised_host: "10.99.48.128",
1385
+ engine_count: 4,
1386
+ ready_timeout: 600s,
1387
+ local_input_address: Some(
1388
+ "<ipc input>",
1389
+ ),
1390
+ local_output_address: Some(
1391
+ "<ipc output>",
1392
+ ),
1393
+ },
1394
+ coordinator_mode: MaybeInProc,
1395
+ model: "Qwen/Qwen3-0.6B",
1396
+ served_model_name: [],
1397
+ listener_mode: BindTcp {
1398
+ host: "127.0.0.1",
1399
+ port: 8000,
1400
+ },
1401
+ tool_call_parser: Auto,
1402
+ reasoning_parser: Auto,
1403
+ renderer: Auto,
1404
+ language_model_only: false,
1405
+ chat_template: None,
1406
+ default_chat_template_kwargs: None,
1407
+ chat_template_content_format: Auto,
1408
+ max_logprobs: None,
1409
+ api_server_options: ApiServerOptions {
1410
+ enable_log_requests: false,
1411
+ enable_prompt_tokens_details: false,
1412
+ enable_request_id_headers: false,
1413
+ },
1414
+ cors: CorsConfig {
1415
+ allow_origins: [
1416
+ "*",
1417
+ ],
1418
+ allow_methods: [
1419
+ "*",
1420
+ ],
1421
+ allow_headers: [
1422
+ "*",
1423
+ ],
1424
+ allow_credentials: false,
1425
+ },
1426
+ tls: None,
1427
+ api_keys: [],
1428
+ disable_log_stats: false,
1429
+ grpc_port: None,
1430
+ shutdown_timeout: 0ns,
1431
+ keep_alive_timeout: 5s,
1432
+ profiler: None,
1433
+ }
1434
+ "#]]
1435
+ .assert_debug_eq(&Config {
1436
+ transport_mode: TransportMode::HandshakeOwner {
1437
+ handshake_address: handshake_address.clone(),
1438
+ advertised_host: advertised_host.clone(),
1439
+ engine_count: *engine_count,
1440
+ ready_timeout: *ready_timeout,
1441
+ local_input_address: Some("<ipc input>".to_string()),
1442
+ local_output_address: Some("<ipc output>".to_string()),
1443
+ },
1444
+ ..config.clone()
1445
+ });
1446
+ }
1447
+
1448
+ #[test]
1449
+ fn serve_frontend_config_keeps_tcp_transport_for_non_local_only_topology() {
1450
+ let cli = Cli::try_parse_from([
1451
+ "vllm-rs",
1452
+ "serve",
1453
+ "Qwen/Qwen3-0.6B",
1454
+ "--data-parallel-address",
1455
+ "10.99.48.128",
1456
+ "--data-parallel-size",
1457
+ "4",
1458
+ "--data-parallel-size-local",
1459
+ "2",
1460
+ ])
1461
+ .unwrap();
1462
+
1463
+ let Command::Serve(args) = cli.command else {
1464
+ panic!("expected serve args");
1465
+ };
1466
+ let config = args.to_frontend_config("tcp://10.99.48.128:29550".to_string());
1467
+
1468
+ expect![[r#"
1469
+ Config {
1470
+ transport_mode: HandshakeOwner {
1471
+ handshake_address: "tcp://10.99.48.128:29550",
1472
+ advertised_host: "10.99.48.128",
1473
+ engine_count: 4,
1474
+ ready_timeout: 600s,
1475
+ local_input_address: None,
1476
+ local_output_address: None,
1477
+ },
1478
+ coordinator_mode: MaybeInProc,
1479
+ model: "Qwen/Qwen3-0.6B",
1480
+ served_model_name: [],
1481
+ listener_mode: BindTcp {
1482
+ host: "127.0.0.1",
1483
+ port: 8000,
1484
+ },
1485
+ tool_call_parser: Auto,
1486
+ reasoning_parser: Auto,
1487
+ renderer: Auto,
1488
+ language_model_only: false,
1489
+ chat_template: None,
1490
+ default_chat_template_kwargs: None,
1491
+ chat_template_content_format: Auto,
1492
+ max_logprobs: None,
1493
+ api_server_options: ApiServerOptions {
1494
+ enable_log_requests: false,
1495
+ enable_prompt_tokens_details: false,
1496
+ enable_request_id_headers: false,
1497
+ },
1498
+ cors: CorsConfig {
1499
+ allow_origins: [
1500
+ "*",
1501
+ ],
1502
+ allow_methods: [
1503
+ "*",
1504
+ ],
1505
+ allow_headers: [
1506
+ "*",
1507
+ ],
1508
+ allow_credentials: false,
1509
+ },
1510
+ tls: None,
1511
+ api_keys: [],
1512
+ disable_log_stats: false,
1513
+ grpc_port: None,
1514
+ shutdown_timeout: 0ns,
1515
+ keep_alive_timeout: 5s,
1516
+ profiler: None,
1517
+ }
1518
+ "#]]
1519
+ .assert_debug_eq(&config);
1520
+ }
1521
+
1522
+ #[test]
1523
+ fn frontend_args_reject_legacy_handshake_flags() {
1524
+ let error = Cli::try_parse_from([
1525
+ "vllm-rs",
1526
+ "frontend",
1527
+ "--listen-fd",
1528
+ "3",
1529
+ "--input-address",
1530
+ "ipc:///tmp/input.sock",
1531
+ "--output-address",
1532
+ "ipc:///tmp/output.sock",
1533
+ "--args-json",
1534
+ r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
1535
+ "--handshake-address",
1536
+ "tcp://127.0.0.1:62100",
1537
+ ])
1538
+ .unwrap_err();
1539
+
1540
+ assert!(error.to_string().contains("--handshake-address"));
1541
+ }
1542
+
1543
+ #[test]
1544
+ fn frontend_config_uses_external_coordinator_when_coordinator_address_is_present() {
1545
+ let cli = Cli::try_parse_from([
1546
+ "vllm-rs",
1547
+ "frontend",
1548
+ "--listen-fd",
1549
+ "3",
1550
+ "--input-address",
1551
+ "ipc:///tmp/input.sock",
1552
+ "--output-address",
1553
+ "ipc:///tmp/output.sock",
1554
+ "--coordinator-address",
1555
+ "tcp://127.0.0.1:7000",
1556
+ "--engine-start-index",
1557
+ "3",
1558
+ "--engine-count",
1559
+ "1",
1560
+ "--args-json",
1561
+ r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
1562
+ ])
1563
+ .unwrap();
1564
+
1565
+ let Command::Frontend(args) = cli.command else {
1566
+ panic!("expected frontend args");
1567
+ };
1568
+ let config = args.into_config();
1569
+
1570
+ expect![[r#"
1571
+ Config {
1572
+ transport_mode: Bootstrapped {
1573
+ input_address: "ipc:///tmp/input.sock",
1574
+ output_address: "ipc:///tmp/output.sock",
1575
+ engine_start_index: 3,
1576
+ engine_count: 1,
1577
+ ready_timeout: 600s,
1578
+ },
1579
+ coordinator_mode: External {
1580
+ address: "tcp://127.0.0.1:7000",
1581
+ },
1582
+ model: "Qwen/Qwen3-0.6B",
1583
+ served_model_name: [],
1584
+ listener_mode: InheritedFd {
1585
+ fd: 3,
1586
+ },
1587
+ tool_call_parser: None,
1588
+ reasoning_parser: None,
1589
+ renderer: Auto,
1590
+ language_model_only: false,
1591
+ chat_template: None,
1592
+ default_chat_template_kwargs: None,
1593
+ chat_template_content_format: Auto,
1594
+ max_logprobs: None,
1595
+ api_server_options: ApiServerOptions {
1596
+ enable_log_requests: false,
1597
+ enable_prompt_tokens_details: false,
1598
+ enable_request_id_headers: false,
1599
+ },
1600
+ cors: CorsConfig {
1601
+ allow_origins: [
1602
+ "*",
1603
+ ],
1604
+ allow_methods: [
1605
+ "*",
1606
+ ],
1607
+ allow_headers: [
1608
+ "*",
1609
+ ],
1610
+ allow_credentials: false,
1611
+ },
1612
+ tls: None,
1613
+ api_keys: [],
1614
+ disable_log_stats: false,
1615
+ grpc_port: None,
1616
+ shutdown_timeout: 0ns,
1617
+ keep_alive_timeout: 5s,
1618
+ profiler: None,
1619
+ }
1620
+ "#]]
1621
+ .assert_debug_eq(&config);
1622
+ }
1623
+
1624
+ #[test]
1625
+ fn serve_frontend_config_uses_unix_listener_when_uds_is_present() {
1626
+ let cli = Cli::try_parse_from([
1627
+ "vllm-rs",
1628
+ "serve",
1629
+ "Qwen/Qwen3-0.6B",
1630
+ "--uds",
1631
+ "/tmp/vllm.sock",
1632
+ ])
1633
+ .unwrap();
1634
+
1635
+ let Command::Serve(args) = cli.command else {
1636
+ panic!("expected serve args");
1637
+ };
1638
+ let config = args.to_frontend_config("tcp://127.0.0.1:29550".to_string());
1639
+
1640
+ assert_eq!(
1641
+ config.listener_mode,
1642
+ HttpListenerMode::BindUnix {
1643
+ path: "/tmp/vllm.sock".to_string(),
1644
+ }
1645
+ );
1646
+ }
1647
+
1648
+ #[test]
1649
+ fn frontend_args_json_enables_profiling_when_profiler_config_set() {
1650
+ let cli = Cli::try_parse_from([
1651
+ "vllm-rs",
1652
+ "frontend",
1653
+ "--listen-fd",
1654
+ "3",
1655
+ "--input-address",
1656
+ "ipc:///tmp/input.sock",
1657
+ "--output-address",
1658
+ "ipc:///tmp/output.sock",
1659
+ "--args-json",
1660
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","profiler_config":{"profiler":"torch","torch_profiler_dir":"/tmp/profile"}}"#,
1661
+ ])
1662
+ .unwrap();
1663
+
1664
+ let Command::Frontend(args) = cli.command else {
1665
+ panic!("expected frontend args");
1666
+ };
1667
+ assert_eq!(args.runtime.profiler().as_deref(), Some("torch"));
1668
+ let config = args.into_config();
1669
+ assert_eq!(config.profiler.as_deref(), Some("torch"));
1670
+ }
1671
+
1672
+ #[test]
1673
+ fn frontend_args_json_disables_profiling_when_profiler_config_absent() {
1674
+ let cli = Cli::try_parse_from([
1675
+ "vllm-rs",
1676
+ "frontend",
1677
+ "--listen-fd",
1678
+ "3",
1679
+ "--input-address",
1680
+ "ipc:///tmp/input.sock",
1681
+ "--output-address",
1682
+ "ipc:///tmp/output.sock",
1683
+ "--args-json",
1684
+ r#"{"model_tag":"Qwen/Qwen3-0.6B"}"#,
1685
+ ])
1686
+ .unwrap();
1687
+
1688
+ let Command::Frontend(args) = cli.command else {
1689
+ panic!("expected frontend args");
1690
+ };
1691
+ assert_eq!(args.runtime.profiler(), None);
1692
+ let config = args.into_config();
1693
+ assert_eq!(config.profiler, None);
1694
+ }
1695
+
1696
+ #[test]
1697
+ fn frontend_args_json_disables_profiling_when_profiler_type_is_null() {
1698
+ let cli = Cli::try_parse_from([
1699
+ "vllm-rs",
1700
+ "frontend",
1701
+ "--listen-fd",
1702
+ "3",
1703
+ "--input-address",
1704
+ "ipc:///tmp/input.sock",
1705
+ "--output-address",
1706
+ "ipc:///tmp/output.sock",
1707
+ "--args-json",
1708
+ r#"{"model_tag":"Qwen/Qwen3-0.6B","profiler_config":{"profiler":null}}"#,
1709
+ ])
1710
+ .unwrap();
1711
+
1712
+ let Command::Frontend(args) = cli.command else {
1713
+ panic!("expected frontend args");
1714
+ };
1715
+ assert_eq!(args.runtime.profiler(), None);
1716
+ let config = args.into_config();
1717
+ assert_eq!(config.profiler, None);
1718
+ }
asian_ai/performance/vLLM/rust/src/cmd/src/cli/unsupported.rs ADDED
@@ -0,0 +1,578 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #![allow(clippy::doc_lazy_continuation)]
2
+
3
+ use std::fmt::Display;
4
+ use std::str::FromStr;
5
+
6
+ use clap::Args;
7
+ use clap::builder::{TypedValueParser, ValueParserFactory};
8
+ use itertools::Itertools;
9
+ use serde::{Deserialize, Deserializer, Serialize};
10
+
11
+ /// Marker type for frontend-owned `serve` arguments that `vllm-rs` recognizes
12
+ /// but does not support yet.
13
+ ///
14
+ /// When passed as JSON args, it can be deserialized from any value, and
15
+ /// serializes back to the original value.
16
+ #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
17
+ #[serde(transparent)]
18
+ pub struct Unsupported(pub serde_json::Value);
19
+
20
+ impl FromStr for Unsupported {
21
+ type Err = String;
22
+
23
+ fn from_str(_s: &str) -> Result<Self, Self::Err> {
24
+ Err("argument is not implemented in Rust frontend yet
25
+
26
+ Remove this unsupported argument to continue.
27
+
28
+ Alternatively, if you intend to pass it only to the Python engine, put it after `--` (e.g., `-- <arg>`).
29
+ This may lead to unexpected behavior as the Rust frontend will completely ignore that argument."
30
+ .to_string())
31
+ }
32
+ }
33
+
34
+ /// Marker type for no-op arguments that are accepted by the Rust frontend but
35
+ /// have no effect.
36
+ ///
37
+ /// When passed as JSON args, it can be deserialized from any value, but always
38
+ /// serializes back to `null`.
39
+ #[derive(Clone, Debug, PartialEq, Eq, Serialize)]
40
+ pub struct Noop;
41
+
42
+ impl<'de> Deserialize<'de> for Noop {
43
+ fn deserialize<D>(_deserializer: D) -> Result<Self, D::Error>
44
+ where
45
+ D: Deserializer<'de>,
46
+ {
47
+ Ok(Noop)
48
+ }
49
+ }
50
+
51
+ impl ValueParserFactory for Noop {
52
+ type Parser = NoopValueParser;
53
+
54
+ fn value_parser() -> Self::Parser {
55
+ NoopValueParser
56
+ }
57
+ }
58
+
59
+ #[derive(Copy, Clone, Debug)]
60
+ pub struct NoopValueParser;
61
+
62
+ #[track_caller]
63
+ fn noop_warn(arg: impl Display) {
64
+ tracing::warn!("argument '{arg}' currently has no effect in Rust frontend, ignoring");
65
+ }
66
+
67
+ impl TypedValueParser for NoopValueParser {
68
+ type Value = Noop;
69
+
70
+ fn parse_ref(
71
+ &self,
72
+ _cmd: &clap::Command,
73
+ arg: Option<&clap::Arg>,
74
+ _value: &std::ffi::OsStr,
75
+ ) -> Result<Self::Value, clap::Error> {
76
+ if let Some(arg) = arg {
77
+ noop_warn(arg);
78
+ }
79
+ Ok(Noop)
80
+ }
81
+ }
82
+
83
+ /// Frontend-owned Python `serve` arguments that `vllm-rs` recognizes but does
84
+ /// not support yet.
85
+ #[serde_with::skip_serializing_none]
86
+ #[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
87
+ #[command(next_help_heading = "Options not implemented in Rust frontend yet")]
88
+ pub struct UnsupportedArgs {
89
+ #[command(flatten)]
90
+ #[serde(default, flatten)]
91
+ top_level: TopLevelUnsupportedArgs,
92
+ #[command(flatten)]
93
+ #[serde(default, flatten)]
94
+ engine: EngineUnsupportedArgs,
95
+ #[command(flatten)]
96
+ #[serde(default, flatten)]
97
+ server: ServerUnsupportedArgs,
98
+ }
99
+
100
+ impl UnsupportedArgs {
101
+ /// Check whether any unsupported arguments are set, and if so, return an
102
+ /// error listing them. Also warn about any no-op arguments that are set
103
+ /// but will be ignored.
104
+ pub(crate) fn check(&self) -> Result<(), String> {
105
+ let value = serde_json::to_value(self).unwrap();
106
+ let map = value.as_object().unwrap();
107
+ let mut unsupported = Vec::new();
108
+
109
+ for (key, value) in map {
110
+ if value.is_null() {
111
+ noop_warn(key);
112
+ } else {
113
+ unsupported.push(key.as_str());
114
+ }
115
+ }
116
+
117
+ if !unsupported.is_empty() {
118
+ unsupported.sort_unstable();
119
+ let bullets = unsupported.into_iter().map(|key| format!("- {key}")).join("\n");
120
+ return Err(format!(
121
+ "
122
+ The following arguments are not implemented in Rust frontend yet:
123
+ {bullets}
124
+
125
+ Remove these arguments to continue."
126
+ ));
127
+ }
128
+
129
+ Ok(())
130
+ }
131
+ }
132
+
133
+ /// Frontend-owned Python `vllm serve` top-level arguments that `vllm-rs`
134
+ /// recognizes but does not support yet.
135
+ ///
136
+ /// Source of truth in Python vLLM:
137
+ /// - `vllm.entrypoints.openai.cli_args.make_arg_parser(...)`
138
+ /// - `vllm.entrypoints.cli.serve.ServeSubcommand.subparser_init(...)`
139
+ ///
140
+ /// These are not part of `EngineArgs`, `AsyncEngineArgs`, `BaseFrontendArgs`,
141
+ /// or `FrontendArgs`. They live on the `serve` command itself and control
142
+ /// managed-engine / multi-process orchestration rather than the shared frontend
143
+ /// runtime config.
144
+ #[serde_with::skip_serializing_none]
145
+ #[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
146
+ pub struct TopLevelUnsupportedArgs {
147
+ /// How many API server processes to run. Defaults to data_parallel_size if
148
+ /// not specified.
149
+ #[arg(long, hide = true)]
150
+ pub api_server_count: Option<Noop>,
151
+
152
+ /// Read CLI options from a config file. Must be a YAML with the following
153
+ /// options: https://docs.vllm.ai/en/latest/configuration/serve_args.html
154
+ #[arg(long)]
155
+ pub config: Option<Unsupported>,
156
+
157
+ /// Launch a gRPC server instead of the HTTP OpenAI-compatible server.
158
+ /// Requires: pip install vllm[grpc].
159
+ #[arg(long, default_missing_value = "true", num_args = 0..=1)]
160
+ pub grpc: Option<Unsupported>,
161
+ }
162
+
163
+ /// Frontend-owned Python engine arguments that `vllm-rs` recognizes but does
164
+ /// not support yet.
165
+ ///
166
+ /// Source of truth in Python vLLM:
167
+ /// - `vllm.engine.arg_utils.EngineArgs.add_cli_args(...)`
168
+ /// - `vllm.engine.arg_utils.AsyncEngineArgs.add_cli_args(...)`
169
+ ///
170
+ /// These arguments are declared through the Python engine-args surface, but
171
+ /// they are still frontend-owned: the API server / AsyncLLM layer reads them
172
+ /// for tokenizer setup, request validation, routing, logging, and other
173
+ /// frontend behavior, so Rust must recognize them rather than treating them as
174
+ /// pure engine passthrough.
175
+ #[serde_with::skip_serializing_none]
176
+ #[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
177
+ pub struct EngineUnsupportedArgs {
178
+ /// Name or path of the Hugging Face tokenizer to use. If unspecified, model
179
+ /// name or path will be used.
180
+ #[arg(long)]
181
+ pub tokenizer: Option<Unsupported>,
182
+
183
+ /// Name or path of the Hugging Face config to use. If unspecified, model
184
+ /// name or path will be used.
185
+ #[arg(long)]
186
+ pub hf_config_path: Option<Unsupported>,
187
+
188
+ /// Allowing API requests to read local images or videos from directories
189
+ /// specified by the server file system. This is a security risk. Should
190
+ /// only be enabled in trusted environments.
191
+ #[arg(long)]
192
+ pub allowed_local_media_path: Option<Unsupported>,
193
+
194
+ /// If set, only media URLs that belong to this domain can be used for
195
+ /// multi-modal inputs.
196
+ #[arg(long)]
197
+ pub allowed_media_domains: Option<Unsupported>,
198
+
199
+ /// The specific revision to use for the tokenizer on the Hugging Face Hub.
200
+ /// It can be a branch name, a tag name, or a commit id. If unspecified,
201
+ /// will use the default version.
202
+ #[arg(long)]
203
+ pub tokenizer_revision: Option<Unsupported>,
204
+
205
+ /// Skip initialization of tokenizer and detokenizer. Expects valid
206
+ /// `prompt_token_ids` and `None` for prompt from the input. The generated
207
+ /// output will contain token ids.
208
+ #[arg(
209
+ long,
210
+ visible_alias = "no-skip-tokenizer-init",
211
+ default_missing_value = "true",
212
+ num_args = 0..=1
213
+ )]
214
+ pub skip_tokenizer_init: Option<Unsupported>,
215
+
216
+ /// If `True`, enables passing text embeddings as inputs via the
217
+ /// `prompt_embeds` key.
218
+ ///
219
+ /// WARNING: The vLLM engine may crash if incorrect shape of embeddings is
220
+ /// passed. Only enable this flag for trusted users!
221
+ #[arg(
222
+ long,
223
+ visible_alias = "no-enable-prompt-embeds",
224
+ default_missing_value = "true",
225
+ num_args = 0..=1
226
+ )]
227
+ pub enable_prompt_embeds: Option<Unsupported>,
228
+
229
+ /// The token to use as HTTP bearer authorization for remote files. If
230
+ /// `True`, will use the token generated when running `hf auth login`
231
+ /// (stored in `~/.cache/huggingface/token`).
232
+ #[arg(long, default_missing_value = "true", num_args = 0..=1)]
233
+ pub hf_token: Option<Unsupported>,
234
+
235
+ /// If a dictionary, contains arguments to be forwarded to the Hugging Face
236
+ /// config. If a callable, it is called to update the HuggingFace config.
237
+ #[arg(long)]
238
+ pub hf_overrides: Option<Unsupported>,
239
+
240
+ /// The folder path to the generation config. Defaults to `"auto"`, the
241
+ /// generation config will be loaded from model path. If set to `"vllm"`, no
242
+ /// generation config is loaded, vLLM defaults will be used. If set to a
243
+ /// folder path, the generation config will be loaded from the specified
244
+ /// folder path. If `max_new_tokens` is specified in generation config,
245
+ /// then it sets a server-wide limit on the number of output tokens for
246
+ /// all requests.
247
+ #[arg(long)]
248
+ pub generation_config: Option<Unsupported>,
249
+
250
+ /// IOProcessor plugin name to load at model startup
251
+ #[arg(long)]
252
+ pub io_processor_plugin: Option<Unsupported>,
253
+
254
+ /// Path to a dynamically reasoning parser plugin that can be dynamically
255
+ /// loaded and registered.
256
+ #[arg(long)]
257
+ pub reasoning_parser_plugin: Option<Unsupported>,
258
+
259
+ /// Rank of the data parallel group.
260
+ #[arg(long, env = "VLLM_DP_RANK")]
261
+ pub data_parallel_rank: Option<Unsupported>,
262
+
263
+ /// Whether to use "hybrid" DP LB mode. Applies only to online serving
264
+ /// and when data_parallel_size > 0. Enables running an AsyncLLM
265
+ /// and API server on a "per-node" basis where vLLM load balances
266
+ /// between local data parallel ranks, but an external LB balances
267
+ /// between vLLM nodes/replicas. Set explicitly in conjunction with
268
+ /// --data-parallel-start-rank.
269
+ #[arg(
270
+ long,
271
+ visible_alias = "no-data-parallel-hybrid-lb",
272
+ default_missing_value = "true",
273
+ num_args = 0..=1
274
+ )]
275
+ pub data_parallel_hybrid_lb: Option<Unsupported>,
276
+
277
+ /// Whether to use "external" DP LB mode. Applies only to online serving
278
+ /// and when data_parallel_size > 0. This is useful for a "one-pod-per-rank"
279
+ /// wide-EP setup in Kubernetes. Set implicitly when --data-parallel-rank
280
+ /// is provided explicitly to vllm serve.
281
+ #[arg(
282
+ long,
283
+ visible_alias = "no-data-parallel-external-lb",
284
+ default_missing_value = "true",
285
+ num_args = 0..=1
286
+ )]
287
+ pub data_parallel_external_lb: Option<Unsupported>,
288
+
289
+ /// This feature is work in progress and no prefill optimization takes place
290
+ /// with this flag enabled currently.
291
+ #[arg(
292
+ long,
293
+ visible_alias = "no-kv-sharing-fast-prefill",
294
+ default_missing_value = "true",
295
+ num_args = 0..=1
296
+ )]
297
+ pub kv_sharing_fast_prefill: Option<Unsupported>,
298
+
299
+ /// The maximum number of input items and options allowed per
300
+ /// prompt for each modality.
301
+ #[arg(long)]
302
+ pub limit_mm_per_prompt: Option<Unsupported>,
303
+
304
+ /// Additional args passed to process media inputs, keyed by modalities.
305
+ #[arg(long)]
306
+ pub media_io_kwargs: Option<Unsupported>,
307
+
308
+ /// Arguments to be forwarded to the model's processor for multi-modal data,
309
+ /// e.g., image processor.
310
+ #[arg(long)]
311
+ pub mm_processor_kwargs: Option<Unsupported>,
312
+
313
+ /// The size (in GiB) of the multi-modal processor cache.
314
+ #[arg(long)]
315
+ pub mm_processor_cache_gb: Option<Unsupported>,
316
+
317
+ /// Type of cache to use for the multi-modal preprocessor/mapper.
318
+ #[arg(long)]
319
+ pub mm_processor_cache_type: Option<Unsupported>,
320
+
321
+ /// Dictionary mapping specific modalities to LoRA model paths.
322
+ #[arg(long)]
323
+ pub default_mm_loras: Option<Unsupported>,
324
+
325
+ /// Target URL to which OpenTelemetry traces will be sent.
326
+ #[arg(long)]
327
+ pub otlp_traces_endpoint: Option<Unsupported>,
328
+
329
+ /// It makes sense to set this only if `--otlp-traces-endpoint` is set.
330
+ #[arg(long)]
331
+ pub collect_detailed_traces: Option<Unsupported>,
332
+
333
+ /// The interval (or buffer size) for streaming in terms of token length.
334
+ #[arg(long)]
335
+ pub stream_interval: Option<Unsupported>,
336
+
337
+ /// Structured outputs configuration.
338
+ #[arg(long)]
339
+ pub structured_outputs_config: Option<Noop>,
340
+
341
+ /// Log aggregate rather than per-engine statistics when using data
342
+ /// parallelism.
343
+ #[arg(long, default_missing_value = "true", num_args = 0..=1)]
344
+ pub aggregate_engine_logging: Option<Unsupported>,
345
+ }
346
+
347
+ /// Frontend-owned Python OpenAI server arguments that `vllm-rs` recognizes but
348
+ /// does not support yet.
349
+ ///
350
+ /// Source of truth in Python vLLM:
351
+ /// - `vllm.entrypoints.openai.cli_args.BaseFrontendArgs`
352
+ /// - `vllm.entrypoints.openai.cli_args.FrontendArgs`
353
+ ///
354
+ /// These are not engine args. They belong to the Python OpenAI-compatible
355
+ /// frontend / API-server layer itself, for example chat-template configuration,
356
+ /// tool/frontend behavior, TLS / CORS / HTTP server settings, and other
357
+ /// northbound server knobs.
358
+ #[serde_with::skip_serializing_none]
359
+ #[derive(Debug, Clone, PartialEq, Eq, Default, Args, Serialize, Deserialize)]
360
+ pub struct ServerUnsupportedArgs {
361
+ /// LoRA modules configurations in either 'name=path' format or JSON format
362
+ /// or JSON list format. Example (old format): `'name=path'` Example (new
363
+ /// format): `{"name": "name", "path": "lora_path",
364
+ /// "base_model_name": "id"}`
365
+ #[arg(long)]
366
+ pub lora_modules: Option<Unsupported>,
367
+
368
+ /// Whether to trust the chat template provided in the request. If False,
369
+ /// the server will always use the chat template specified by
370
+ /// `--chat-template` or the ones from tokenizer.
371
+ #[arg(
372
+ long,
373
+ visible_alias = "no-trust-request-chat-template",
374
+ default_missing_value = "true",
375
+ num_args = 0..=1
376
+ )]
377
+ pub trust_request_chat_template: Option<Unsupported>,
378
+
379
+ /// The role name to return if `request.add_generation_prompt=true`.
380
+ #[arg(long)]
381
+ pub response_role: Option<Unsupported>,
382
+
383
+ /// When `--max-logprobs` is specified, represents single tokens as
384
+ /// strings of the form 'token_id:{token_id}' so that tokens that are not
385
+ /// JSON-encodable can be identified.
386
+ #[arg(
387
+ long,
388
+ visible_alias = "no-return-tokens-as-token-ids",
389
+ default_missing_value = "true",
390
+ num_args = 0..=1
391
+ )]
392
+ pub return_tokens_as_token_ids: Option<Unsupported>,
393
+
394
+ /// Enable auto tool choice for supported models. Use `--tool-call-parser`
395
+ /// to specify which parser to use.
396
+ #[arg(
397
+ long,
398
+ visible_alias = "no-enable-auto-tool-choice",
399
+ default_missing_value = "true",
400
+ num_args = 0..=1,
401
+ hide = true
402
+ )]
403
+ pub enable_auto_tool_choice: Option<Noop>,
404
+
405
+ /// If specified, exclude tool definitions in prompts when
406
+ /// tool_choice='none'.
407
+ #[arg(
408
+ long,
409
+ visible_alias = "no-exclude-tools-when-tool-choice-none",
410
+ default_missing_value = "true",
411
+ num_args = 0..=1
412
+ )]
413
+ pub exclude_tools_when_tool_choice_none: Option<Unsupported>,
414
+
415
+ /// Special the tool parser plugin write to parse the model-generated tool
416
+ /// into OpenAI API format, the name register in this plugin can be used in
417
+ /// `--tool-call-parser`.
418
+ #[arg(long)]
419
+ pub tool_parser_plugin: Option<Unsupported>,
420
+
421
+ /// Comma-separated list of host:port pairs (IPv4, IPv6, or hostname).
422
+ /// Examples: 127.0.0.1:8000, [::1]:8000, localhost:1234. Or `demo` for
423
+ /// built-in demo tools (browser and Python code interpreter). WARNING:
424
+ /// The `demo` Python tool executes model-generated code in Docker without
425
+ /// network isolation by default. See the security guide for more
426
+ /// information.
427
+ #[arg(long)]
428
+ pub tool_server: Option<Unsupported>,
429
+
430
+ /// Path to logging config JSON file for both vllm and uvicorn
431
+ #[arg(long, /* env = "VLLM_LOGGING_CONFIG_PATH" */)]
432
+ pub log_config_file: Option<Unsupported>,
433
+
434
+ /// Max number of prompt characters or prompt ID numbers being printed in
435
+ /// log. The default of None means unlimited.
436
+ #[arg(long)]
437
+ pub max_log_len: Option<Unsupported>,
438
+
439
+ /// If set to True, enable tracking server_load_metrics in the app state.
440
+ #[arg(
441
+ long,
442
+ visible_alias = "no-enable-server-load-tracking",
443
+ default_missing_value = "true",
444
+ num_args = 0..=1
445
+ )]
446
+ pub enable_server_load_tracking: Option<Noop>,
447
+
448
+ /// If set to True, including usage on every request.
449
+ #[arg(
450
+ long,
451
+ visible_alias = "no-enable-force-include-usage",
452
+ default_missing_value = "true",
453
+ num_args = 0..=1
454
+ )]
455
+ pub enable_force_include_usage: Option<Unsupported>,
456
+
457
+ /// Enable the `/tokenizer_info` endpoint. May expose chat
458
+ /// templates and other tokenizer configuration.
459
+ #[arg(
460
+ long,
461
+ visible_alias = "no-enable-tokenizer-info-endpoint",
462
+ default_missing_value = "true",
463
+ num_args = 0..=1
464
+ )]
465
+ pub enable_tokenizer_info_endpoint: Option<Unsupported>,
466
+
467
+ /// If set to True, log model outputs (generations).
468
+ /// Requires `--enable-log-requests`. As with `--enable-log-requests`,
469
+ /// information is only logged at INFO level at maximum.
470
+ #[arg(
471
+ long,
472
+ visible_alias = "no-enable-log-outputs",
473
+ default_missing_value = "true",
474
+ num_args = 0..=1
475
+ )]
476
+ pub enable_log_outputs: Option<Unsupported>,
477
+
478
+ /// If set to False, output deltas will not be logged. Relevant only if
479
+ /// --enable-log-outputs is set.
480
+ #[arg(
481
+ long,
482
+ visible_alias = "no-enable-log-deltas",
483
+ default_missing_value = "true",
484
+ num_args = 0..=1
485
+ )]
486
+ pub enable_log_deltas: Option<Unsupported>,
487
+
488
+ /// If set to True, log the stack trace of error responses
489
+ #[arg(
490
+ long,
491
+ // env = "VLLM_SERVER_DEV_MODE",
492
+ visible_alias = "no-log-error-stack",
493
+ default_missing_value = "true",
494
+ num_args = 0..=1
495
+ )]
496
+ pub log_error_stack: Option<Unsupported>,
497
+
498
+ /// If set to True, only enable the Tokens In<>Out endpoint.
499
+ /// This is intended for use in a Disaggregated Everything setup.
500
+ #[arg(
501
+ long,
502
+ visible_alias = "no-tokens-only",
503
+ default_missing_value = "true",
504
+ num_args = 0..=1
505
+ )]
506
+ pub tokens_only: Option<Unsupported>,
507
+
508
+ /// Log level for uvicorn.
509
+ #[arg(long)]
510
+ pub uvicorn_log_level: Option<Unsupported>,
511
+
512
+ /// Disable uvicorn access log.
513
+ #[arg(
514
+ long,
515
+ visible_alias = "no-disable-uvicorn-access-log",
516
+ default_missing_value = "true",
517
+ num_args = 0..=1
518
+ )]
519
+ pub disable_uvicorn_access_log: Option<Noop>,
520
+
521
+ /// Comma-separated list of endpoint paths to exclude from uvicorn access
522
+ /// logs. This is useful to reduce log noise from high-frequency endpoints
523
+ /// like health checks. Example: "/health,/metrics,/ping".
524
+ /// When set, access logs for requests to these paths will be suppressed
525
+ /// while keeping logs for other endpoints.
526
+ #[arg(long)]
527
+ pub disable_access_log_for_endpoints: Option<Noop>,
528
+
529
+ /// Refresh SSL Context when SSL certificate files change
530
+ #[arg(
531
+ long,
532
+ visible_alias = "no-enable-ssl-refresh",
533
+ default_missing_value = "true",
534
+ num_args = 0..=1
535
+ )]
536
+ pub enable_ssl_refresh: Option<Unsupported>,
537
+
538
+ /// FastAPI root_path when app is behind a path based routing proxy.
539
+ #[arg(long)]
540
+ pub root_path: Option<Unsupported>,
541
+
542
+ /// Additional ASGI middleware to apply to the app. We accept multiple
543
+ /// --middleware arguments. The value should be an import path. If a
544
+ /// function is provided, vLLM will add it to the server using
545
+ /// `@app.middleware('http')`. If a class is provided, vLLM will
546
+ /// add it to the server using `app.add_middleware()`.
547
+ #[arg(long)]
548
+ pub middleware: Option<Unsupported>,
549
+
550
+ /// Disable FastAPI's OpenAPI schema, Swagger UI, and ReDoc endpoint.
551
+ #[arg(
552
+ long,
553
+ visible_alias = "no-disable-fastapi-docs",
554
+ default_missing_value = "true",
555
+ num_args = 0..=1
556
+ )]
557
+ pub disable_fastapi_docs: Option<Unsupported>,
558
+
559
+ /// Maximum size (bytes) of an incomplete HTTP event (header or body) for
560
+ /// h11 parser. Helps mitigate header abuse. Default: 4194304 (4 MB).
561
+ #[arg(long)]
562
+ pub h11_max_incomplete_event_size: Option<Unsupported>,
563
+
564
+ /// Maximum number of HTTP headers allowed in a request for h11 parser.
565
+ /// Helps mitigate header abuse. Default: 256.
566
+ #[arg(long)]
567
+ pub h11_max_header_count: Option<Unsupported>,
568
+
569
+ /// Enable offline FastAPI documentation for air-gapped environments.
570
+ /// Uses vendored static assets bundled with vLLM.
571
+ #[arg(
572
+ long,
573
+ visible_alias = "no-enable-offline-docs",
574
+ default_missing_value = "true",
575
+ num_args = 0..=1
576
+ )]
577
+ pub enable_offline_docs: Option<Unsupported>,
578
+ }
asian_ai/performance/vLLM/rust/src/cmd/src/logging.rs ADDED
@@ -0,0 +1,328 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::{env, fmt, process};
2
+
3
+ use time::UtcOffset;
4
+ use time::macros::format_description;
5
+ use tracing::level_filters::LevelFilter;
6
+ use tracing::{Event, Level, Subscriber};
7
+ use tracing_subscriber::Layer as _;
8
+ use tracing_subscriber::filter::Targets;
9
+ use tracing_subscriber::fmt::format::{FormatEvent, FormatFields, Writer};
10
+ use tracing_subscriber::fmt::time::FormatTime;
11
+ use tracing_subscriber::fmt::{FmtContext, FormattedFields};
12
+ use tracing_subscriber::layer::SubscriberExt as _;
13
+ use tracing_subscriber::registry::LookupSpan;
14
+ use tracing_subscriber::util::SubscriberInitExt as _;
15
+
16
+ const CYAN: &str = "\x1b[0;36m";
17
+ const GREY: &str = "\x1b[90m";
18
+ const GREEN: &str = "\x1b[32m";
19
+ const YELLOW: &str = "\x1b[33m";
20
+ const RED: &str = "\x1b[31m";
21
+ const WHITE: &str = "\x1b[37m";
22
+ const RESET: &str = "\x1b[0m";
23
+ const VLLM_TIME_FORMAT: &[time::format_description::FormatItem<'static>] =
24
+ format_description!("[month]-[day] [hour]:[minute]:[second]");
25
+
26
+ const PROCESS_LABEL: &str = "RustFrontend";
27
+
28
+ /// Install the process-wide vLLM-style tracing subscriber for the CLI binary.
29
+ pub(crate) fn init_tracing() {
30
+ let filter = build_targets_filter(
31
+ env::var("VLLM_LOGGING_LEVEL").ok().as_deref(),
32
+ env::var("RUST_LOG").ok().as_deref(),
33
+ );
34
+ let formatter = VllmEventFormatter::new();
35
+
36
+ let _ = tracing_subscriber::registry()
37
+ .with(tracing_subscriber::fmt::layer().event_format(formatter).with_filter(filter))
38
+ .try_init();
39
+ }
40
+
41
+ /// Build the CLI log filter by merging the vLLM-style default level with
42
+ /// Rust-style target overrides.
43
+ ///
44
+ /// Precedence:
45
+ /// - Start from `VLLM_LOGGING_LEVEL` as the default level for all targets.
46
+ /// - If `RUST_LOG` contains a global default level such as `warn`, it overrides
47
+ /// `VLLM_LOGGING_LEVEL`.
48
+ /// - Any explicit target directives in `RUST_LOG`, such as `hyper=info`, override whichever default
49
+ /// level is active for those targets only.
50
+ fn build_targets_filter(vllm_logging_level: Option<&str>, rust_log: Option<&str>) -> Targets {
51
+ let mut filter =
52
+ Targets::new().with_default(map_python_log_level(vllm_logging_level.unwrap_or("INFO")));
53
+
54
+ if let Some(rust_log) = rust_log
55
+ && !rust_log.is_empty()
56
+ {
57
+ let rust_log_targets: Targets = rust_log.parse().expect("failed to parse `RUST_LOG`");
58
+ if let Some(default_level) = rust_log_targets.default_level() {
59
+ filter = filter.with_default(default_level);
60
+ }
61
+ filter = filter.with_targets(rust_log_targets);
62
+ }
63
+
64
+ filter
65
+ }
66
+
67
+ #[derive(Debug, Clone, Copy)]
68
+ struct VllmLocalTimer {
69
+ local_offset: UtcOffset,
70
+ }
71
+
72
+ impl Default for VllmLocalTimer {
73
+ fn default() -> Self {
74
+ let local_offset = UtcOffset::current_local_offset().unwrap_or(UtcOffset::UTC);
75
+ Self { local_offset }
76
+ }
77
+ }
78
+
79
+ impl FormatTime for VllmLocalTimer {
80
+ fn format_time(&self, w: &mut Writer<'_>) -> fmt::Result {
81
+ let now = time::OffsetDateTime::now_utc().to_offset(self.local_offset);
82
+ let formatted = now.format(VLLM_TIME_FORMAT).map_err(|_| fmt::Error)?;
83
+ w.write_str(&formatted)
84
+ }
85
+ }
86
+
87
+ #[derive(Debug, Clone)]
88
+ struct VllmEventFormatter {
89
+ prefix: String,
90
+ timer: VllmLocalTimer,
91
+ }
92
+
93
+ impl VllmEventFormatter {
94
+ fn new() -> Self {
95
+ Self {
96
+ prefix: format!("({} pid={})", PROCESS_LABEL, process::id()),
97
+ timer: VllmLocalTimer::default(),
98
+ }
99
+ }
100
+
101
+ fn write_process_prefix(&self, writer: &mut Writer<'_>, ansi: bool) -> fmt::Result {
102
+ write_colored(writer, ansi, Some(CYAN), &self.prefix)?;
103
+ writer.write_char(' ')
104
+ }
105
+
106
+ fn write_level(&self, writer: &mut Writer<'_>, level: &Level, ansi: bool) -> fmt::Result {
107
+ let (text, color) = match *level {
108
+ Level::TRACE => ("TRACE", WHITE),
109
+ Level::DEBUG => ("DEBUG", WHITE),
110
+ Level::INFO => ("INFO", GREEN),
111
+ Level::WARN => ("WARNING", YELLOW),
112
+ Level::ERROR => ("ERROR", RED),
113
+ };
114
+ write_colored(writer, ansi, Some(color), text)
115
+ }
116
+
117
+ fn write_timestamp(&self, writer: &mut Writer<'_>, ansi: bool) -> fmt::Result {
118
+ if ansi {
119
+ writer.write_str(GREY)?;
120
+ }
121
+ if self.timer.format_time(writer).is_err() {
122
+ writer.write_str("<unknown time>")?;
123
+ }
124
+ if ansi {
125
+ writer.write_str(RESET)?;
126
+ }
127
+ Ok(())
128
+ }
129
+
130
+ fn write_location(
131
+ &self,
132
+ writer: &mut Writer<'_>,
133
+ file: Option<&str>,
134
+ line: Option<u32>,
135
+ full_path: bool,
136
+ ansi: bool,
137
+ ) -> fmt::Result {
138
+ let Some(file) = file else {
139
+ return Ok(());
140
+ };
141
+ let file = if full_path {
142
+ file
143
+ } else {
144
+ shorten_file_path(file)
145
+ };
146
+ if ansi {
147
+ writer.write_str(GREY)?;
148
+ }
149
+ match line {
150
+ Some(line) => write!(writer, "[{file}:{line}]")?,
151
+ None => write!(writer, "[{file}]")?,
152
+ }
153
+ if ansi {
154
+ writer.write_str(RESET)?;
155
+ }
156
+ Ok(())
157
+ }
158
+
159
+ fn write_scope<S, N>(&self, ctx: &FmtContext<'_, S, N>, writer: &mut Writer<'_>) -> fmt::Result
160
+ where
161
+ S: Subscriber + for<'lookup> LookupSpan<'lookup>,
162
+ N: for<'writer> FormatFields<'writer> + 'static,
163
+ {
164
+ let Some(scope) = ctx.event_scope() else {
165
+ return Ok(());
166
+ };
167
+
168
+ let mut seen = false;
169
+ for span in scope.from_root() {
170
+ if seen {
171
+ writer.write_str(":")?;
172
+ }
173
+ seen = true;
174
+ writer.write_str(span.metadata().name())?;
175
+
176
+ let ext = span.extensions();
177
+ if let Some(fields) = ext.get::<FormattedFields<N>>()
178
+ && !fields.is_empty()
179
+ {
180
+ write!(writer, "{{{fields}}}")?;
181
+ }
182
+ }
183
+
184
+ if seen {
185
+ writer.write_str(": ")?;
186
+ }
187
+
188
+ Ok(())
189
+ }
190
+ }
191
+
192
+ impl<S, N> FormatEvent<S, N> for VllmEventFormatter
193
+ where
194
+ S: Subscriber + for<'lookup> LookupSpan<'lookup>,
195
+ N: for<'writer> FormatFields<'writer> + 'static,
196
+ {
197
+ fn format_event(
198
+ &self,
199
+ ctx: &FmtContext<'_, S, N>,
200
+ mut writer: Writer<'_>,
201
+ event: &Event<'_>,
202
+ ) -> fmt::Result {
203
+ let meta = event.metadata();
204
+ let ansi = writer.has_ansi_escapes();
205
+
206
+ self.write_process_prefix(&mut writer, ansi)?;
207
+ self.write_level(&mut writer, meta.level(), ansi)?;
208
+ writer.write_char(' ')?;
209
+ self.write_timestamp(&mut writer, ansi)?;
210
+ writer.write_char(' ')?;
211
+ // Use the full file path only when DEBUG (or more verbose) is enabled anywhere,
212
+ // independent of the level of this particular event. Filenames alone are often
213
+ // ambiguous, but full paths are too noisy for normal INFO-level operation.
214
+ let full_path = LevelFilter::current() >= LevelFilter::DEBUG;
215
+ self.write_location(&mut writer, meta.file(), meta.line(), full_path, ansi)?;
216
+ writer.write_char(' ')?;
217
+ self.write_scope(ctx, &mut writer)?;
218
+ ctx.format_fields(writer.by_ref(), event)?;
219
+ writer.write_char('\n')
220
+ }
221
+ }
222
+
223
+ /// Shorten a source file path for log output while preserving enough context
224
+ /// for common Rust entrypoint and module filenames.
225
+ ///
226
+ /// - For `mod.rs`, keep the parent directory as `parent/mod.rs`.
227
+ /// - For `src/lib.rs` and `src/main.rs`, keep one additional component as `crate/src/lib.rs` or
228
+ /// `crate/src/main.rs` when available.
229
+ /// - Other files are displayed as just the basename.
230
+ fn shorten_file_path(file: &str) -> &str {
231
+ let mut parts = file.rsplit('/');
232
+ let name = parts.next().unwrap_or(file);
233
+ let parent = parts.next();
234
+ let grandparent = parts.next();
235
+
236
+ let Some(parent) = parent else {
237
+ return file;
238
+ };
239
+
240
+ if name == "mod.rs" {
241
+ return &file[file.len() - parent.len() - 1 - name.len()..];
242
+ }
243
+
244
+ if !matches!(name, "lib.rs" | "main.rs") || parent != "src" {
245
+ return name;
246
+ }
247
+ let Some(grandparent) = grandparent else {
248
+ return file;
249
+ };
250
+
251
+ &file[file.len() - grandparent.len() - 1 - parent.len() - 1 - name.len()..]
252
+ }
253
+
254
+ fn write_colored(
255
+ writer: &mut Writer<'_>,
256
+ ansi: bool,
257
+ color: Option<&str>,
258
+ text: &str,
259
+ ) -> fmt::Result {
260
+ if ansi {
261
+ if let Some(color) = color {
262
+ writer.write_str(color)?;
263
+ }
264
+ writer.write_str(text)?;
265
+ if color.is_some() {
266
+ writer.write_str(RESET)?;
267
+ }
268
+ return Ok(());
269
+ }
270
+
271
+ writer.write_str(text)
272
+ }
273
+
274
+ /// Map a Python logging level name to the corresponding Rust tracing level.
275
+ fn map_python_log_level(level: &str) -> LevelFilter {
276
+ match level.to_ascii_uppercase().as_str() {
277
+ "CRITICAL" | "FATAL" => LevelFilter::ERROR,
278
+ "ERROR" => LevelFilter::ERROR,
279
+ "WARNING" | "WARN" => LevelFilter::WARN,
280
+ "INFO" => LevelFilter::INFO,
281
+ "DEBUG" => LevelFilter::DEBUG,
282
+ "NOTSET" => LevelFilter::TRACE,
283
+ _ => LevelFilter::INFO,
284
+ }
285
+ }
286
+
287
+ #[cfg(test)]
288
+ mod tests {
289
+ use super::*;
290
+
291
+ #[test]
292
+ fn rust_log_target_overrides_are_merged_with_vllm_default_level() {
293
+ let filter = build_targets_filter(Some("DEBUG"), Some("hyper=warn,tower=error"));
294
+
295
+ assert_eq!(filter.to_string(), "tower=error,hyper=warn,debug");
296
+ }
297
+
298
+ #[test]
299
+ fn rust_log_default_level_overrides_vllm_default_level() {
300
+ let filter = build_targets_filter(Some("DEBUG"), Some("warn,hyper=info"));
301
+
302
+ assert_eq!(filter.to_string(), "hyper=info,warn");
303
+ }
304
+
305
+ #[test]
306
+ fn invalid_vllm_level_falls_back_to_info() {
307
+ let filter = build_targets_filter(Some("bogus"), None);
308
+
309
+ assert_eq!(filter.to_string(), "info");
310
+ }
311
+
312
+ #[test]
313
+ fn location_path_uses_filename_for_non_ambiguous_files() {
314
+ assert_eq!(shorten_file_path("src/cmd/src/logging.rs"), "logging.rs");
315
+ assert_eq!(shorten_file_path("src/chat/lib.rs"), "lib.rs");
316
+ assert_eq!(shorten_file_path("src/chat/main.rs"), "main.rs");
317
+ assert_eq!(shorten_file_path("src/chat/src/xmod.rs"), "xmod.rs");
318
+ }
319
+
320
+ #[test]
321
+ fn location_path_keeps_more_context_for_common_entrypoint_filenames() {
322
+ assert_eq!(shorten_file_path("src/lib.rs"), "src/lib.rs");
323
+ assert_eq!(shorten_file_path("src/chat/src/lib.rs"), "chat/src/lib.rs");
324
+ assert_eq!(shorten_file_path("src/cmd/src/main.rs"), "cmd/src/main.rs");
325
+ assert_eq!(shorten_file_path("mod.rs"), "mod.rs");
326
+ assert_eq!(shorten_file_path("src/chat/src/tool/mod.rs"), "tool/mod.rs");
327
+ }
328
+ }
asian_ai/performance/vLLM/rust/src/cmd/src/main.rs ADDED
@@ -0,0 +1,192 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ mod cli;
2
+ mod logging;
3
+
4
+ use std::env;
5
+ use std::process::ExitStatus;
6
+
7
+ use anyhow::{Context, Result, anyhow, bail};
8
+ use tokio_util::sync::CancellationToken;
9
+ use tracing::{info, warn};
10
+ use vllm_managed_engine::ManagedEngineHandle;
11
+
12
+ use crate::cli::{Cli, Command};
13
+
14
+ #[global_allocator]
15
+ static GLOBAL: mimalloc::MiMalloc = mimalloc::MiMalloc;
16
+
17
+ const TOKIO_WORKER_THREADS_ENV: &str = "TOKIO_WORKER_THREADS";
18
+ const DEFAULT_MAX_TOKIO_WORKER_THREADS: usize = 32;
19
+
20
+ /// Cap the default number of Tokio worker threads if the user did not
21
+ /// explicitly set `TOKIO_WORKER_THREADS` to avoid spawning too many threads on
22
+ /// machines with a large number of CPUs, which may lead to excessive context
23
+ /// switching and degraded performance.
24
+ fn tokio_worker_threads() -> Option<usize> {
25
+ if env::var_os(TOKIO_WORKER_THREADS_ENV).is_some() {
26
+ return None;
27
+ }
28
+
29
+ std::thread::available_parallelism()
30
+ .map(|parallelism| {
31
+ let available = parallelism.get();
32
+ let worker_threads = available.min(DEFAULT_MAX_TOKIO_WORKER_THREADS);
33
+ if worker_threads < available {
34
+ info!(
35
+ available_parallelism = available,
36
+ capped_worker_threads = worker_threads,
37
+ "capping tokio worker threads, set {TOKIO_WORKER_THREADS_ENV} to override"
38
+ );
39
+ }
40
+ worker_threads
41
+ })
42
+ .ok()
43
+ }
44
+
45
+ /// Reason that caused a managed `serve` session to stop.
46
+ #[derive(Debug)]
47
+ enum ShutdownReason {
48
+ Signal,
49
+ Server(anyhow::Error),
50
+ EngineExited(ExitStatus),
51
+ }
52
+
53
+ /// Cancellation token tripped by Ctrl-C or SIGTERM.
54
+ fn shutdown_signal() -> CancellationToken {
55
+ let token = CancellationToken::new();
56
+ let shutdown = token.clone();
57
+
58
+ tokio::spawn(async move {
59
+ let ctrl_c = async {
60
+ tokio::signal::ctrl_c().await.expect("failed to install Ctrl-C signal handler");
61
+ };
62
+
63
+ let sigterm = async {
64
+ tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
65
+ .expect("failed to install SIGTERM signal handler")
66
+ .recv()
67
+ .await;
68
+ };
69
+
70
+ tokio::select! {
71
+ _ = ctrl_c => info!("received shutdown signal (Ctrl-C), shutting down..."),
72
+ _ = sigterm => info!("received shutdown signal (SIGTERM), shutting down..."),
73
+ }
74
+
75
+ shutdown.cancel();
76
+ });
77
+
78
+ token
79
+ }
80
+
81
+ fn main() -> Result<()> {
82
+ logging::init_tracing();
83
+ let cli = Cli::parse();
84
+
85
+ let mut runtime = tokio::runtime::Builder::new_multi_thread();
86
+ runtime.enable_all();
87
+ if let Some(worker_threads) = tokio_worker_threads() {
88
+ runtime.worker_threads(worker_threads);
89
+ }
90
+
91
+ runtime
92
+ .build()
93
+ .context("failed to build Tokio runtime")?
94
+ .block_on(async_main(cli))
95
+ }
96
+
97
+ async fn async_main(cli: Cli) -> Result<()> {
98
+ match cli.command {
99
+ Command::Frontend(args) => vllm_server::serve(args.into_config(), shutdown_signal()).await,
100
+ Command::Serve(args) => {
101
+ let handshake_port = args.managed_engine.resolve_handshake_port()?;
102
+
103
+ if args.managed_engine.data_parallel_size_local == Some(0) {
104
+ if args.headless {
105
+ bail!("cannot combine `--headless` with `--data-parallel-size-local 0`");
106
+ }
107
+
108
+ let handshake_address = args.managed_engine.handshake_address(handshake_port);
109
+ info!(
110
+ %handshake_address,
111
+ engine_count = args.managed_engine.data_parallel_size,
112
+ "running Rust frontend without a managed local Python engine"
113
+ );
114
+ let config = args.to_frontend_config(handshake_address);
115
+ return vllm_server::serve(config, shutdown_signal()).await;
116
+ }
117
+
118
+ let shutdown_timeout = args.runtime.shutdown_timeout();
119
+ let engine_config = args.to_managed_engine_config(handshake_port);
120
+ let handshake_address = engine_config.handshake_address();
121
+
122
+ let engine = ManagedEngineHandle::spawn(engine_config)
123
+ .await
124
+ .context("failed to start managed Python headless engine")?;
125
+
126
+ let shutdown = shutdown_signal();
127
+
128
+ let mut serve_task = if args.headless {
129
+ info!("running managed Python headless engine without Rust frontend");
130
+ let shutdown = shutdown.clone();
131
+ tokio::spawn(async move {
132
+ shutdown.cancelled().await;
133
+ Ok(())
134
+ })
135
+ } else {
136
+ let config = args.to_frontend_config(handshake_address);
137
+ let shutdown = shutdown.clone();
138
+ tokio::spawn(async move {
139
+ let result = vllm_server::serve(config, shutdown).await;
140
+ if result.is_ok() {
141
+ info!("OpenAI server shut down gracefully");
142
+ }
143
+ result
144
+ })
145
+ };
146
+
147
+ let shutdown_reason = tokio::select! {
148
+ biased;
149
+
150
+ // Received shutdown signal via Ctrl-C or SIGTERM.
151
+ _ = shutdown.cancelled() => ShutdownReason::Signal,
152
+
153
+ // Engine process exited unexpectedly.
154
+ status = engine.wait_for_exit() => {
155
+ warn!(%status, "managed Python headless engine exited, shutting down...");
156
+ ShutdownReason::EngineExited(status)
157
+ }
158
+
159
+ // Serve task exited unexpectedly.
160
+ serve_result = &mut serve_task => {
161
+ let serve_result = serve_result.context("serve task join failed")?;
162
+ match serve_result {
163
+ Ok(()) => ShutdownReason::Server(anyhow!("OpenAI server shut down unexpectedly without error")),
164
+ Err(error) => ShutdownReason::Server(error),
165
+ }
166
+ }
167
+ };
168
+ // Regardless of the shutdown reason, broadcast shutdown signal here to ensure
169
+ // that all serving tasks are notified.
170
+ shutdown.cancel();
171
+
172
+ // Shutdown begins. Terminate the managed engine first.
173
+ engine.shutdown(shutdown_timeout).await?;
174
+ info!("managed engine shut down gracefully");
175
+ // Wait for the API server to shut down gracefully by draining in-flight
176
+ // requests.
177
+ if !matches!(shutdown_reason, ShutdownReason::Server(_)) {
178
+ serve_task.await.context("serve task join failed")??;
179
+ }
180
+
181
+ match shutdown_reason {
182
+ ShutdownReason::Signal => Ok(()),
183
+ ShutdownReason::Server(error) => {
184
+ Err(error.context("OpenAI server shut down unexpectedly"))
185
+ }
186
+ ShutdownReason::EngineExited(status) => Err(anyhow!(
187
+ "managed Python headless engine exited unexpectedly with status {status}"
188
+ )),
189
+ }
190
+ }
191
+ }
192
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/Cargo.toml ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ [package]
2
+ name = "vllm-engine-core-client"
3
+ version.workspace = true
4
+ edition.workspace = true
5
+ license.workspace = true
6
+
7
+ [features]
8
+ test-util = ["dep:tempfile"]
9
+
10
+ [dependencies]
11
+ arc-swap.workspace = true
12
+ bytemuck.workspace = true
13
+ byteorder.workspace = true
14
+ bytes.workspace = true
15
+ easy-ext.workspace = true
16
+ enum-as-inner.workspace = true
17
+ futures.workspace = true
18
+ half.workspace = true
19
+ hex.workspace = true
20
+ itertools.workspace = true
21
+ parking_lot.workspace = true
22
+ rmp-serde.workspace = true
23
+ rmpv.workspace = true
24
+ serde.workspace = true
25
+ serde_default.workspace = true
26
+ serde_json.workspace = true
27
+ serde_repr.workspace = true
28
+ serde_tuple.workspace = true
29
+ serde_with.workspace = true
30
+ task-local.workspace = true
31
+ tempfile = { workspace = true, optional = true }
32
+ thiserror.workspace = true
33
+ thiserror-ext.workspace = true
34
+ tokio.workspace = true
35
+ tokio-util.workspace = true
36
+ tracing.workspace = true
37
+ vllm-metrics.workspace = true
38
+ zeromq.workspace = true
39
+
40
+ [dev-dependencies]
41
+ anyhow.workspace = true
42
+ clap.workspace = true
43
+ expect-test.workspace = true
44
+ hex.workspace = true
45
+ tempfile.workspace = true
46
+ tracing-subscriber.workspace = true
47
+
48
+ [lints]
49
+ workspace = true
asian_ai/performance/vLLM/rust/src/engine-core-client/examples/README.md ADDED
@@ -0,0 +1,53 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # Engine-Core Smoke Tests
2
+
3
+ Start headless `vllm`:
4
+
5
+ ```bash
6
+ source ../vllm/.venv/bin/activate
7
+ HF_HUB_OFFLINE=1 \
8
+ VLLM_LOGGING_LEVEL=DEBUG \
9
+ VLLM_CPU_KVCACHE_SPACE=2 \
10
+ VLLM_HOST_IP=127.0.0.1 \
11
+ VLLM_LOOPBACK_IP=127.0.0.1 \
12
+ python3 -m vllm.entrypoints.cli.main serve Qwen/Qwen3-0.6B \
13
+ --headless \
14
+ --enable-sleep-mode \
15
+ --data-parallel-address 127.0.0.1 \
16
+ --data-parallel-rpc-port 62100 \
17
+ --data-parallel-size-local 1 \
18
+ --max-model-len 512 \
19
+ --dtype float16
20
+ ```
21
+
22
+ Run the Rust smoke test through the `vllm-engine-core-client` utility interface:
23
+
24
+ ```bash
25
+ cargo run -p vllm-engine-core-client --example external_engine_utility_call -- \
26
+ --handshake-address tcp://127.0.0.1:62100 \
27
+ --host 127.0.0.1
28
+ ```
29
+
30
+ If your current engine setup does not support sleep mode, skip the `sleep` / `wake_up` part of the
31
+ smoke:
32
+
33
+ ```bash
34
+ cargo run -p vllm-engine-core-client --example external_engine_utility_call -- \
35
+ --handshake-address tcp://127.0.0.1:62100 \
36
+ --host 127.0.0.1 \
37
+ --skip-sleep-wake
38
+ ```
39
+
40
+ Run the Rust smoke test for sample logprobs decoding through the raw engine-core request path:
41
+
42
+ ```bash
43
+ cargo run -p vllm-engine-core-client --example external_engine_logprobs -- \
44
+ --handshake-address tcp://127.0.0.1:62100 \
45
+ --host 127.0.0.1
46
+ ```
47
+
48
+ This smoke requests a small generated-token `logprobs` payload plus prompt logprobs over a much
49
+ longer prompt, so it exercises both the inline and aux-frame decode paths against a real engine.
50
+ The Rust client decodes those payloads into semantic per-position records rather than exposing the
51
+ raw ndarray/tensor wire shape.
52
+
53
+ IMPORTANT: You must restart `vllm` each time you run the smoke test, as the vLLM engine cannot manage frontend closures and subsequent reconnects. In other words, do not reuse existing `vllm` instances, if any.
asian_ai/performance/vLLM/rust/src/engine-core-client/examples/external_engine_logprobs.rs ADDED
@@ -0,0 +1,190 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::time::Duration;
2
+
3
+ use anyhow::{Context, Result, bail};
4
+ use clap::Parser;
5
+ use futures::StreamExt as _;
6
+ use tokio::time::timeout;
7
+ use tracing_subscriber::EnvFilter;
8
+ use vllm_engine_core_client::protocol::output::EngineCoreFinishReason;
9
+ use vllm_engine_core_client::protocol::request::EngineCoreRequest;
10
+ use vllm_engine_core_client::protocol::sampling::EngineCoreSamplingParams;
11
+ use vllm_engine_core_client::{
12
+ EngineCoreClient, EngineCoreClientConfig, EngineCoreStreamOutput, TransportMode,
13
+ };
14
+
15
+ const BASE_PROMPT_TOKEN_IDS: &[u32] = &[20841, 448, 6896, 25, 23811];
16
+
17
+ #[derive(Debug, Parser)]
18
+ #[command(about = "Smoke-test engine-core sample logprobs against an external vLLM engine.")]
19
+ struct Args {
20
+ #[arg(long)]
21
+ handshake_address: String,
22
+ #[arg(long, default_value_t = 1)]
23
+ engine_count: usize,
24
+ #[arg(long, default_value = "Qwen/Qwen3-0.6B")]
25
+ model: String,
26
+ #[arg(long, default_value = "127.0.0.1")]
27
+ host: String,
28
+ #[arg(long, default_value_t = 0)]
29
+ client_index: u32,
30
+ #[arg(long, default_value_t = 30)]
31
+ ready_timeout_secs: u64,
32
+ #[arg(long, default_value_t = 120)]
33
+ output_timeout_secs: u64,
34
+ #[arg(long, default_value_t = 1)]
35
+ max_tokens: u32,
36
+ #[arg(long, default_value_t = 2)]
37
+ logprobs: i32,
38
+ #[arg(long, default_value_t = 1)]
39
+ prompt_logprobs: i32,
40
+ #[arg(long, default_value_t = 96)]
41
+ prompt_repeats: usize,
42
+ }
43
+
44
+ fn init_tracing() {
45
+ let filter = EnvFilter::try_from_default_env()
46
+ .unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
47
+ let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
48
+ }
49
+
50
+ fn unique_request_id() -> String {
51
+ let nanos = std::time::SystemTime::now()
52
+ .duration_since(std::time::UNIX_EPOCH)
53
+ .expect("system clock should be after unix epoch")
54
+ .as_nanos();
55
+ format!("rust-engine-core-logprobs-{nanos}")
56
+ }
57
+
58
+ fn build_prompt_token_ids(prompt_repeats: usize) -> Vec<u32> {
59
+ let repeats = prompt_repeats.max(1);
60
+ BASE_PROMPT_TOKEN_IDS.repeat(repeats)
61
+ }
62
+
63
+ fn build_request(
64
+ request_id: String,
65
+ prompt_token_ids: Vec<u32>,
66
+ max_tokens: u32,
67
+ logprobs: i32,
68
+ prompt_logprobs: i32,
69
+ client_index: u32,
70
+ ) -> EngineCoreRequest {
71
+ EngineCoreRequest {
72
+ request_id,
73
+ prompt_token_ids: Some(prompt_token_ids),
74
+ sampling_params: Some(EngineCoreSamplingParams {
75
+ max_tokens,
76
+ logprobs: Some(logprobs),
77
+ prompt_logprobs: Some(prompt_logprobs),
78
+ ..EngineCoreSamplingParams::for_test()
79
+ }),
80
+ arrival_time: 0.0,
81
+ client_index,
82
+ ..EngineCoreRequest::default()
83
+ }
84
+ }
85
+
86
+ async fn wait_for_final_output(
87
+ mut stream: vllm_engine_core_client::EngineCoreOutputStream,
88
+ ) -> Result<EngineCoreStreamOutput> {
89
+ while let Some(output) = stream.next().await {
90
+ let output = output.context("failed to receive engine-core output")?;
91
+ if output.finished() {
92
+ return Ok(output);
93
+ }
94
+ }
95
+ bail!("request stream ended without a final output")
96
+ }
97
+
98
+ #[tokio::main(flavor = "multi_thread")]
99
+ async fn main() -> Result<()> {
100
+ init_tracing();
101
+ let args = Args::parse();
102
+ let ready_timeout = Duration::from_secs(args.ready_timeout_secs);
103
+ let output_timeout = Duration::from_secs(args.output_timeout_secs);
104
+ let request_id = unique_request_id();
105
+ let prompt_token_ids = build_prompt_token_ids(args.prompt_repeats);
106
+ let client = EngineCoreClient::connect(EngineCoreClientConfig {
107
+ transport_mode: TransportMode::HandshakeOwner {
108
+ handshake_address: args.handshake_address.clone(),
109
+ advertised_host: args.host.clone(),
110
+ engine_count: args.engine_count,
111
+ ready_timeout,
112
+ local_input_address: None,
113
+ local_output_address: None,
114
+ },
115
+ coordinator_mode: None,
116
+ model_name: args.model.clone(),
117
+ client_index: args.client_index,
118
+ })
119
+ .await
120
+ .context("failed to connect to external vLLM engine")?;
121
+
122
+ println!("model={}", args.model);
123
+ println!("handshake_address={}", args.handshake_address);
124
+ println!("engine_count={}", args.engine_count);
125
+ println!("input_address={}", client.input_address());
126
+ println!("output_address={}", client.output_address());
127
+ println!("engine_identities={:x?}", client.engine_identities());
128
+
129
+ let request = build_request(
130
+ request_id.clone(),
131
+ prompt_token_ids.clone(),
132
+ args.max_tokens,
133
+ args.logprobs,
134
+ args.prompt_logprobs,
135
+ args.client_index,
136
+ );
137
+ println!("request_id={request_id}");
138
+ println!("prompt_len={}", prompt_token_ids.len());
139
+ println!("base_prompt_len={}", BASE_PROMPT_TOKEN_IDS.len());
140
+ println!("prompt_repeats={}", args.prompt_repeats);
141
+ println!("requested_logprobs={}", args.logprobs);
142
+ println!("requested_prompt_logprobs={}", args.prompt_logprobs);
143
+
144
+ let stream = client.call(request).await.context("failed to submit engine-core request")?;
145
+ let output = timeout(output_timeout, wait_for_final_output(stream))
146
+ .await
147
+ .context("timed out waiting for final output")??;
148
+
149
+ let finish_reason = output.finish_reason;
150
+ let token_ids = output.new_token_ids.clone();
151
+ let logprobs = output
152
+ .new_logprobs
153
+ .as_ref()
154
+ .and_then(|value| value.as_direct())
155
+ .context("engine output did not include decoded sample logprobs")?;
156
+ let prompt_logprobs = output
157
+ .new_prompt_logprobs_tensors
158
+ .as_ref()
159
+ .and_then(|value| value.as_direct())
160
+ .context("engine output did not include decoded prompt logprobs")?;
161
+
162
+ println!("token_ids={token_ids:?}");
163
+ println!("finish_reason={finish_reason:?}");
164
+ println!("new_logprobs={logprobs:#?}");
165
+ println!("new_prompt_logprobs_tensors={prompt_logprobs:#?}");
166
+
167
+ client.shutdown().await.context("failed to shut down engine-core client")?;
168
+
169
+ if finish_reason != Some(EngineCoreFinishReason::Length) {
170
+ bail!("unexpected finish_reason: expected Length, got {finish_reason:?}");
171
+ }
172
+ if token_ids.is_empty() {
173
+ bail!("engine returned no generated token ids");
174
+ }
175
+ if logprobs.is_empty() {
176
+ bail!("decoded logprobs payload is unexpectedly empty");
177
+ }
178
+ if prompt_logprobs.is_empty() {
179
+ bail!("decoded prompt logprobs payload is unexpectedly empty");
180
+ }
181
+ if prompt_logprobs.len() + 1 < prompt_token_ids.len() {
182
+ bail!(
183
+ "prompt logprobs rows look too short: prompt_len={}, rows={}",
184
+ prompt_token_ids.len(),
185
+ prompt_logprobs.len()
186
+ );
187
+ }
188
+
189
+ Ok(())
190
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/examples/external_engine_utility_call.rs ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::time::Duration;
2
+
3
+ use anyhow::{Context, Result, bail};
4
+ use clap::Parser;
5
+ use tracing_subscriber::EnvFilter;
6
+ use vllm_engine_core_client::protocol::utility::PauseMode;
7
+ use vllm_engine_core_client::{EngineCoreClient, EngineCoreClientConfig, TransportMode};
8
+
9
+ #[derive(Debug, Parser)]
10
+ #[command(about = "Smoke-test EngineCoreClient utility calls against an external vLLM engine.")]
11
+ struct Args {
12
+ #[arg(long)]
13
+ handshake_address: String,
14
+ #[arg(long, default_value_t = 1)]
15
+ engine_count: usize,
16
+ #[arg(long, default_value = "Qwen/Qwen3-0.6B")]
17
+ model: String,
18
+ #[arg(long, default_value = "127.0.0.1")]
19
+ host: String,
20
+ #[arg(long, default_value_t = 0)]
21
+ client_index: u32,
22
+ #[arg(long, default_value_t = 30)]
23
+ ready_timeout_secs: u64,
24
+ #[arg(
25
+ long,
26
+ default_value_t = false,
27
+ help = "Expected initial result of is_sleeping() before running the smoke steps."
28
+ )]
29
+ expected_is_sleeping: bool,
30
+ #[arg(long, default_value_t = false)]
31
+ reset_running_requests: bool,
32
+ #[arg(long, default_value_t = false)]
33
+ reset_external: bool,
34
+ #[arg(long, default_value_t = 1)]
35
+ sleep_level: u32,
36
+ #[arg(long, default_value_t = PauseMode::Abort)]
37
+ sleep_mode: PauseMode,
38
+ #[arg(
39
+ long,
40
+ default_value_t = false,
41
+ help = "Skip sleep/wake_up calls when the engine was not started with sleep-mode support."
42
+ )]
43
+ skip_sleep_wake: bool,
44
+ }
45
+
46
+ fn init_tracing() {
47
+ let filter = EnvFilter::try_from_default_env()
48
+ .unwrap_or_else(|_| EnvFilter::new("vllm_engine_core_client=debug"));
49
+ let _ = tracing_subscriber::fmt().with_env_filter(filter).try_init();
50
+ }
51
+
52
+ #[tokio::main(flavor = "multi_thread")]
53
+ async fn main() -> Result<()> {
54
+ init_tracing();
55
+ let args = Args::parse();
56
+ let client = EngineCoreClient::connect(EngineCoreClientConfig {
57
+ transport_mode: TransportMode::HandshakeOwner {
58
+ handshake_address: args.handshake_address.clone(),
59
+ advertised_host: args.host.clone(),
60
+ engine_count: args.engine_count,
61
+ ready_timeout: Duration::from_secs(args.ready_timeout_secs),
62
+ local_input_address: None,
63
+ local_output_address: None,
64
+ },
65
+ coordinator_mode: None,
66
+ model_name: args.model.clone(),
67
+ client_index: args.client_index,
68
+ })
69
+ .await
70
+ .context("failed to connect to external vLLM engine")?;
71
+
72
+ println!("model={}", args.model);
73
+ println!("handshake_address={}", args.handshake_address);
74
+ println!("engine_count={}", args.engine_count);
75
+ println!("input_address={}", client.input_address());
76
+ println!("output_address={}", client.output_address());
77
+ println!("engine_identities={:x?}", client.engine_identities());
78
+
79
+ let initial_is_sleeping =
80
+ client.is_sleeping().await.context("failed to call is_sleeping utility")?;
81
+
82
+ println!("is_sleeping(initial)={initial_is_sleeping}");
83
+
84
+ if initial_is_sleeping != args.expected_is_sleeping {
85
+ bail!(
86
+ "unexpected initial is_sleeping state: expected {}, got {}",
87
+ args.expected_is_sleeping,
88
+ initial_is_sleeping
89
+ );
90
+ }
91
+
92
+ let reset_prefix_cache = client
93
+ .reset_prefix_cache(args.reset_running_requests, args.reset_external)
94
+ .await
95
+ .context("failed to call reset_prefix_cache utility")?;
96
+ println!("reset_prefix_cache={reset_prefix_cache}");
97
+
98
+ client.reset_mm_cache().await.context("failed to call reset_mm_cache utility")?;
99
+ println!("reset_mm_cache=ok");
100
+
101
+ client
102
+ .reset_encoder_cache()
103
+ .await
104
+ .context("failed to call reset_encoder_cache utility")?;
105
+ println!("reset_encoder_cache=ok");
106
+
107
+ if args.skip_sleep_wake {
108
+ println!("sleep_wake=skipped");
109
+ } else {
110
+ client.sleep(args.sleep_level, args.sleep_mode).await.with_context(|| {
111
+ format!(
112
+ "failed to call sleep utility with level={} mode={}",
113
+ args.sleep_level, args.sleep_mode
114
+ )
115
+ })?;
116
+ println!(
117
+ "sleep=ok level={} mode={}",
118
+ args.sleep_level, args.sleep_mode
119
+ );
120
+
121
+ let sleeping_after_sleep =
122
+ client.is_sleeping().await.context("failed to call is_sleeping after sleep")?;
123
+ println!("is_sleeping(after_sleep)={sleeping_after_sleep}");
124
+
125
+ if !sleeping_after_sleep {
126
+ bail!("engine should report sleeping=true after sleep()");
127
+ }
128
+
129
+ client.wake_up(None).await.context("failed to call wake_up utility")?;
130
+ println!("wake_up=ok");
131
+
132
+ let sleeping_after_wake =
133
+ client.is_sleeping().await.context("failed to call is_sleeping after wake_up")?;
134
+ println!("is_sleeping(after_wake)={sleeping_after_wake}");
135
+
136
+ if sleeping_after_wake {
137
+ bail!("engine should report sleeping=false after wake_up()");
138
+ }
139
+ }
140
+
141
+ client.shutdown().await.context("failed to shut down engine-core client")?;
142
+
143
+ Ok(())
144
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/client.rs ADDED
@@ -0,0 +1,793 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::sync::Arc;
2
+ use std::time::Duration;
3
+
4
+ use futures::future::{join_all, try_join_all};
5
+ use itertools::Itertools;
6
+ use serde::Serialize;
7
+ use tokio::sync::mpsc;
8
+ use tokio_util::task::AbortOnDropHandle;
9
+ use tracing::{debug, info, trace};
10
+
11
+ use crate::client::imp::{ClientInner, run_abort_loop, run_output_dispatcher_loop};
12
+ use crate::coordinator::CoordinatorHandle;
13
+ use crate::error::{Error, Result};
14
+ use crate::protocol::dtype::ModelDtype;
15
+ use crate::protocol::handshake::EngineCoreReadyResponse;
16
+ use crate::protocol::lora::LoraRequest;
17
+ use crate::protocol::request::{EngineCoreRequest, EngineCoreRequestType};
18
+ use crate::protocol::utility::{EngineCoreUtilityRequest, PauseMode};
19
+ use crate::runtime::{BackgroundShutdownRuntime, build_zmq_runtime};
20
+ use crate::transport::{self, ConnectedEngine};
21
+
22
+ pub(crate) mod imp;
23
+ mod state;
24
+ mod stream;
25
+
26
+ pub use stream::{EngineCoreOutputStream, EngineCoreStreamOutput};
27
+
28
+ /// How the frontend acquires its request/response transport with Python
29
+ /// `EngineCoreProc`s.
30
+ #[derive(Debug, Clone, PartialEq, Eq, Serialize)]
31
+ pub enum TransportMode {
32
+ /// The Rust process owns the startup handshake and allocates or binds the
33
+ /// frontend transport addresses itself before replying to engine
34
+ /// `HELLO` messages.
35
+ HandshakeOwner {
36
+ /// Shared handshake endpoint that engines dial during startup.
37
+ handshake_address: String,
38
+ /// Host/IP that engines should use to connect back to the frontend
39
+ /// transport sockets.
40
+ advertised_host: String,
41
+ /// Total number of engines expected to join this transport.
42
+ engine_count: usize,
43
+ /// Maximum time to wait for each startup phase to complete.
44
+ ready_timeout: Duration,
45
+ /// Optional explicit bind address for the input ROUTER socket.
46
+ local_input_address: Option<String>,
47
+ /// Optional explicit bind address for the output PULL socket.
48
+ local_output_address: Option<String>,
49
+ },
50
+
51
+ /// The Python supervisor has already chosen the frontend transport
52
+ /// addresses, and the Rust process only needs to bind them and wait for
53
+ /// engine registration frames.
54
+ Bootstrapped {
55
+ /// Input ROUTER socket address that engines will connect to for
56
+ /// requests.
57
+ input_address: String,
58
+ /// Output PULL socket address that engines will connect to for
59
+ /// responses.
60
+ output_address: String,
61
+ /// First data-parallel engine rank expected to register on this
62
+ /// transport.
63
+ engine_start_index: u32,
64
+ /// Total number of engines expected to register on this transport.
65
+ engine_count: usize,
66
+ /// Maximum time to wait for all expected engines to register.
67
+ ready_timeout: Duration,
68
+ },
69
+ }
70
+
71
+ /// Which coordinator implementation should be active when one is present for a
72
+ /// frontend client.
73
+ #[derive(Debug, Clone, PartialEq, Eq)]
74
+ pub enum CoordinatorMode {
75
+ /// Run the Rust in-process coordinator for managed `serve` deployments.
76
+ InProc,
77
+ /// Connect to an external coordinator owned by another process.
78
+ External { address: String },
79
+ }
80
+
81
+ /// Configuration for connecting a Rust frontend client to an already running
82
+ /// Python `EngineCoreProc`.
83
+ #[derive(Debug, Clone, PartialEq, Eq)]
84
+ pub struct EngineCoreClientConfig {
85
+ /// Frontend-to-engine transport setup.
86
+ pub transport_mode: TransportMode,
87
+ /// Frontend-side coordinator behavior, or `None` when requests should flow
88
+ /// directly to engines without any coordinator involvement.
89
+ pub coordinator_mode: Option<CoordinatorMode>,
90
+ /// Model name used for frontend-side metrics labels.
91
+ pub model_name: String,
92
+ /// Frontend client index stamped onto every request.
93
+ pub client_index: u32,
94
+ }
95
+
96
+ impl EngineCoreClientConfig {
97
+ /// Create a new client config with the given handshake address, expecting a
98
+ /// single engine, and default values for all other fields.
99
+ pub fn new_single(handshake_address: impl Into<String>) -> Self {
100
+ Self {
101
+ transport_mode: TransportMode::HandshakeOwner {
102
+ handshake_address: handshake_address.into(),
103
+ advertised_host: "127.0.0.1".to_string(),
104
+ engine_count: 1,
105
+ ready_timeout: Duration::from_secs(30),
106
+ local_input_address: None,
107
+ local_output_address: None,
108
+ },
109
+ coordinator_mode: None,
110
+ model_name: String::new(),
111
+ client_index: 0,
112
+ }
113
+ }
114
+
115
+ /// Set the model name used by frontend-side metrics and diagnostics.
116
+ pub fn with_model_name(mut self, model_name: impl Into<String>) -> Self {
117
+ self.model_name = model_name.into();
118
+ self
119
+ }
120
+
121
+ /// Override the client index stamped onto every outgoing request.
122
+ pub fn with_client_index(mut self, client_index: u32) -> Self {
123
+ self.client_index = client_index;
124
+ self
125
+ }
126
+
127
+ /// Override the optional coordinator mode for this client config.
128
+ pub fn with_coordinator_mode(mut self, coordinator_mode: Option<CoordinatorMode>) -> Self {
129
+ self.coordinator_mode = coordinator_mode;
130
+ self
131
+ }
132
+
133
+ /// Override the locally bound input/output addresses for handshake-owned
134
+ /// transport mode.
135
+ ///
136
+ /// This is primarily used by tests that want deterministic IPC endpoints
137
+ /// while still exercising the handshake-owned startup path.
138
+ pub fn with_local_input_output_addresses(
139
+ mut self,
140
+ local_input_address: Option<String>,
141
+ local_output_address: Option<String>,
142
+ ) -> Self {
143
+ let TransportMode::HandshakeOwner {
144
+ local_input_address: current_input,
145
+ local_output_address: current_output,
146
+ ..
147
+ } = &mut self.transport_mode
148
+ else {
149
+ panic!("local input/output overrides are only valid in handshake-owned mode");
150
+ };
151
+ *current_input = local_input_address;
152
+ *current_output = local_output_address;
153
+ self
154
+ }
155
+ }
156
+
157
+ /// The reason a request stream is being aborted when its output stream is
158
+ /// dropped.
159
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
160
+ pub enum AbortCause {
161
+ /// The consumer dropped the stream before the request reached a terminal
162
+ /// engine output.
163
+ #[default]
164
+ DroppedStream,
165
+ /// The frontend matched a stop string locally and intentionally stopped
166
+ /// consuming the stream.
167
+ StopStringMatched,
168
+ }
169
+
170
+ task_local::task_local! {
171
+ static ABORT_CAUSE: AbortCause;
172
+ }
173
+
174
+ impl AbortCause {
175
+ /// Return the abort cause currently associated with this task, or
176
+ /// [`AbortCause::DroppedStream`] by default.
177
+ pub fn current() -> Self {
178
+ ABORT_CAUSE.try_get().unwrap_or_default()
179
+ }
180
+
181
+ /// Drop one value while marking the drop as happening for this abort cause.
182
+ pub fn drop_as<T>(self, value: T) {
183
+ ABORT_CAUSE.sync_scope(self, move || drop(value));
184
+ }
185
+ }
186
+
187
+ /// Internal auto-abort work item sent from stream `Drop` handlers to the abort
188
+ /// worker.
189
+ #[derive(Debug, Clone)]
190
+ pub(crate) struct AbortRequest {
191
+ request_id: String,
192
+ cause: AbortCause,
193
+ }
194
+
195
+ /// Default ZMQ-based implementation that talks directly to a Python
196
+ /// `EngineCoreProc`.
197
+ pub struct EngineCoreClient {
198
+ config: EngineCoreClientConfig,
199
+ input_address: String,
200
+ output_address: String,
201
+ engines: Vec<ConnectedEngine>,
202
+ inner: Arc<ClientInner>,
203
+ coordinator: Option<CoordinatorHandle>,
204
+ abort_tx: mpsc::UnboundedSender<AbortRequest>,
205
+
206
+ /// Runtime used to send messages to the engine and drive all background tasks.
207
+ runtime: BackgroundShutdownRuntime,
208
+ // Background tasks
209
+ output_task: AbortOnDropHandle<()>,
210
+ dispatcher_task: AbortOnDropHandle<()>,
211
+ abort_task: AbortOnDropHandle<()>,
212
+ coordinator_output_task: Option<AbortOnDropHandle<()>>,
213
+ coordinator_task: Option<AbortOnDropHandle<()>>,
214
+ }
215
+
216
+ impl EngineCoreClient {
217
+ /// Connect to Python `EngineCoreProc`s using the configured
218
+ /// transport/coordinator modes.
219
+ ///
220
+ /// In handshake-owned mode this method drives the full engine startup
221
+ /// handshake. In bootstrapped mode it binds the provided frontend
222
+ /// sockets and waits for the expected engine registration frames.
223
+ pub async fn connect(config: EngineCoreClientConfig) -> Result<Self> {
224
+ let connected = match &config.transport_mode {
225
+ TransportMode::HandshakeOwner {
226
+ handshake_address,
227
+ advertised_host,
228
+ engine_count,
229
+ ready_timeout,
230
+ local_input_address,
231
+ local_output_address,
232
+ } => {
233
+ let enable_inproc_coordinator = match config.coordinator_mode {
234
+ None => false,
235
+ Some(CoordinatorMode::InProc) => true,
236
+ Some(CoordinatorMode::External { .. }) => {
237
+ return Err(Error::UnsupportedExternalCoordinator);
238
+ }
239
+ };
240
+
241
+ transport::connect_handshake(
242
+ handshake_address,
243
+ *engine_count,
244
+ advertised_host,
245
+ local_input_address.as_deref(),
246
+ local_output_address.as_deref(),
247
+ enable_inproc_coordinator,
248
+ *ready_timeout,
249
+ )
250
+ .await?
251
+ }
252
+
253
+ TransportMode::Bootstrapped {
254
+ input_address,
255
+ output_address,
256
+ engine_start_index,
257
+ engine_count,
258
+ ready_timeout,
259
+ } => {
260
+ if let Some(CoordinatorMode::InProc) = config.coordinator_mode {
261
+ panic!("cannot use in-process coordinator with bootstrapped transport mode")
262
+ }
263
+
264
+ transport::connect_bootstrapped(
265
+ input_address,
266
+ output_address,
267
+ *engine_start_index,
268
+ *engine_count,
269
+ *ready_timeout,
270
+ )
271
+ .await?
272
+ }
273
+ };
274
+
275
+ Self::from_connected(config, connected).await
276
+ }
277
+
278
+ /// Create a new client instance from the connected transport state after
279
+ /// the startup handshake completes.
280
+ async fn from_connected(
281
+ config: EngineCoreClientConfig,
282
+ connected: transport::ConnectedTransport,
283
+ ) -> Result<Self> {
284
+ let (output_tx, output_rx) = mpsc::channel(64);
285
+ let (abort_tx, abort_rx) = mpsc::unbounded_channel();
286
+ let engines = connected.engines;
287
+ let runtime = build_zmq_runtime();
288
+ let inner = Arc::new(ClientInner::new(
289
+ connected.input_send,
290
+ runtime.handle().clone(),
291
+ config.model_name.clone(),
292
+ &engines,
293
+ ));
294
+ let output_task = AbortOnDropHandle::new(runtime.spawn(transport::run_output_loop(
295
+ connected.output_socket,
296
+ output_tx,
297
+ )));
298
+ let dispatcher_task = AbortOnDropHandle::new(
299
+ runtime.spawn(run_output_dispatcher_loop(inner.clone(), output_rx)),
300
+ );
301
+ let abort_task =
302
+ AbortOnDropHandle::new(runtime.spawn(run_abort_loop(inner.clone(), abort_rx)));
303
+
304
+ // If any engine reported a dp_stats_address in its ready response, use it
305
+ // as the external coordinator address.
306
+ let dp_stats_address: Option<String> =
307
+ engines.iter().find_map(|engine| engine.ready_response.dp_stats_address.clone());
308
+
309
+ let (coordinator, coordinator_output_task, coordinator_task) =
310
+ if let Some(coordinator_transport) = connected.coordinator {
311
+ let (handle, runner) =
312
+ CoordinatorHandle::new_inproc(coordinator_transport.input_socket);
313
+ let (coordinator_output_tx, coordinator_output_rx) = mpsc::channel(64);
314
+ let coordinator_output_task =
315
+ AbortOnDropHandle::new(runtime.spawn(transport::run_output_loop(
316
+ coordinator_transport.output_socket,
317
+ coordinator_output_tx,
318
+ )));
319
+ let coordinator_task = AbortOnDropHandle::new(
320
+ runtime.spawn(runner.run(coordinator_output_rx, inner.clone())),
321
+ );
322
+ (
323
+ Some(handle),
324
+ Some(coordinator_output_task),
325
+ Some(coordinator_task),
326
+ )
327
+ } else if let Some(address) =
328
+ dp_stats_address.as_deref().or(match config.coordinator_mode.as_ref() {
329
+ Some(CoordinatorMode::External { address }) => Some(address.as_str()),
330
+ _ => None,
331
+ })
332
+ {
333
+ let (handle, service) = CoordinatorHandle::connect_external(address).await?;
334
+ let coordinator_task =
335
+ AbortOnDropHandle::new(runtime.spawn(service.run(inner.clone())));
336
+ (Some(handle), None, Some(coordinator_task))
337
+ } else {
338
+ (None, None, None)
339
+ };
340
+
341
+ Ok(Self {
342
+ config,
343
+ input_address: connected.input_address,
344
+ output_address: connected.output_address,
345
+ engines,
346
+ inner,
347
+ coordinator,
348
+ abort_tx,
349
+ runtime,
350
+ output_task,
351
+ dispatcher_task,
352
+ abort_task,
353
+ coordinator_output_task,
354
+ coordinator_task,
355
+ })
356
+ }
357
+
358
+ /// Return the address of the input socket that the client uses to send
359
+ /// requests to the engine.
360
+ pub fn input_address(&self) -> &str {
361
+ &self.input_address
362
+ }
363
+
364
+ /// Return the address of the output socket that the client listens on for
365
+ /// engine responses.
366
+ pub fn output_address(&self) -> &str {
367
+ &self.output_address
368
+ }
369
+
370
+ /// Return the number of engines connected to this client.
371
+ pub fn engine_count(&self) -> usize {
372
+ self.engines.len()
373
+ }
374
+
375
+ /// Return the engine-side indices connected to this client.
376
+ pub fn engine_indices(&self) -> Vec<u32> {
377
+ self.engines
378
+ .iter()
379
+ .map(|engine| engine.engine_id.engine_index().expect("engine id must encode as u16"))
380
+ .collect()
381
+ }
382
+
383
+ /// Return the engine identities of all engines connected to this client.
384
+ pub fn engine_identities(&self) -> Vec<&[u8]> {
385
+ self.engines.iter().map(|engine| &*engine.engine_id).collect()
386
+ }
387
+
388
+ /// Return the ready responses received from all engines on the input
389
+ /// socket.
390
+ pub fn ready_responses(&self) -> Vec<&EngineCoreReadyResponse> {
391
+ self.engines.iter().map(|engine| &engine.ready_response).collect()
392
+ }
393
+
394
+ /// Return the engine-reported effective model dtype.
395
+ pub fn model_dtype(&self) -> ModelDtype {
396
+ self.engines
397
+ .first()
398
+ .expect("engine core client requires at least one engine")
399
+ .ready_response
400
+ .dtype
401
+ }
402
+
403
+ /// Return the engine-reported Python vLLM version.
404
+ pub fn vllm_version(&self) -> &str {
405
+ self.engines
406
+ .first()
407
+ .expect("engine core client requires at least one engine")
408
+ .ready_response
409
+ .vllm_version
410
+ .as_str()
411
+ }
412
+
413
+ /// Return the total number of GPU blocks summed across all connected
414
+ /// engines.
415
+ pub fn total_num_gpu_blocks(&self) -> u64 {
416
+ self.engines.iter().map(|engine| engine.ready_response.num_gpu_blocks).sum()
417
+ }
418
+
419
+ /// Return the minimum engine-reported `max_model_len` across all engines.
420
+ ///
421
+ /// This is the auto-fitted value after KV cache profiling and may differ
422
+ /// from the originally configured value.
423
+ pub fn max_model_len(&self) -> u32 {
424
+ self.engines
425
+ .iter()
426
+ .map(|engine| engine.ready_response.max_model_len as u32)
427
+ .min()
428
+ .expect("engine core client requires at least one engine")
429
+ }
430
+
431
+ /// Return the world size (TP * PP) from the parallel config, if available.
432
+ pub fn world_size(&self) -> u64 {
433
+ self.engines
434
+ .first()
435
+ .expect("engine core client requires at least one engine")
436
+ .ready_response
437
+ .world_size
438
+ }
439
+
440
+ /// Return the data parallel size from the parallel config, if available.
441
+ pub fn data_parallel_size(&self) -> u64 {
442
+ self.engines
443
+ .first()
444
+ .expect("engine core client requires at least one engine")
445
+ .ready_response
446
+ .data_parallel_size
447
+ }
448
+
449
+ /// Get the model name associated with this client used for metrics
450
+ /// labeling.
451
+ pub fn model_name(&self) -> &str {
452
+ self.inner.model_name()
453
+ }
454
+
455
+ /// Return whether the client still considers the engine healthy.
456
+ pub fn is_healthy(&self) -> bool {
457
+ self.inner.is_healthy()
458
+ }
459
+
460
+ /// Return the first persistent health error observed by the client, if any.
461
+ pub fn health_error(&self) -> Option<Arc<Error>> {
462
+ self.inner.health_error()
463
+ }
464
+ }
465
+
466
+ // Client API implementation.
467
+ impl EngineCoreClient {
468
+ /// Add a new request to the engine and return a per-request raw output
469
+ /// stream.
470
+ pub async fn call(&self, mut req: EngineCoreRequest) -> Result<EngineCoreOutputStream> {
471
+ req.client_index = self.config.client_index;
472
+ req.validate()?;
473
+ trace!(
474
+ request_id = %req.request_id,
475
+ client_index = req.client_index,
476
+ current_wave = req.current_wave,
477
+ request = ?req,
478
+ "sending add request"
479
+ );
480
+
481
+ let request_id = req.request_id.clone();
482
+ let lora_name = req.lora_request.as_ref().map(|lora| lora.lora_name.clone());
483
+ let data_parallel_rank = req.data_parallel_rank;
484
+ let (engine_id, rx) =
485
+ self.inner.register_request(request_id.clone(), lora_name, data_parallel_rank)?;
486
+
487
+ let result: Result<()> = async {
488
+ if let Some(coordinator) = self.coordinator.as_ref() {
489
+ let snapshot = coordinator.snapshot();
490
+ req.current_wave = snapshot.current_wave;
491
+ if !snapshot.engines_running {
492
+ coordinator.notify_first_request(engine_id.clone())?;
493
+ }
494
+ }
495
+
496
+ debug!(
497
+ request_id = req.request_id,
498
+ ?engine_id,
499
+ "registered request to engine"
500
+ );
501
+
502
+ self.inner.send_to_engine(&engine_id, EngineCoreRequestType::Add, &req).await?;
503
+ Ok(())
504
+ }
505
+ .await;
506
+
507
+ // Failed to send the request to the engine, roll back the registration.
508
+ if let Err(error) = result {
509
+ self.inner.rollback_request(&request_id);
510
+ return Err(error);
511
+ }
512
+
513
+ Ok(EngineCoreOutputStream::new(
514
+ request_id,
515
+ self.abort_tx.clone(),
516
+ rx,
517
+ ))
518
+ }
519
+
520
+ /// Abort currently in-flight requests by request ID.
521
+ pub async fn abort(&self, ids: &[String]) -> Result<()> {
522
+ let abortable = self.inner.abortable_request_ids(ids)?;
523
+
524
+ trace!(request_ids = ?ids, abortable_request_ids = ?abortable, "sending abort request ids");
525
+
526
+ if abortable.is_empty() {
527
+ return Ok(());
528
+ }
529
+
530
+ // Finalize the consumer streams first, before the engine round-trip.
531
+ let all_request_ids: Vec<String> = abortable.values().flatten().cloned().collect();
532
+ self.inner.abort_requests_locally(&all_request_ids);
533
+
534
+ for (engine_id, request_ids) in abortable {
535
+ self.inner.do_abort_requests(&engine_id, &request_ids).await?;
536
+ }
537
+ Ok(())
538
+ }
539
+
540
+ /// Call a typed utility method on all connected engines, returning one
541
+ /// decoded result per connected engine if all calls succeed or an error
542
+ /// if any call fails.
543
+ ///
544
+ /// Callers should pass utility arguments using Rust tuple semantics so the
545
+ /// encoded payload matches Python's `(client_index, call_id,
546
+ /// method_name, args)` contract: `()`, `(arg,)`, `(arg1, arg2)`, etc.
547
+ pub async fn call_utility<T, A>(&self, method: &str, args: A) -> Result<Vec<T>>
548
+ where
549
+ T: serde::de::DeserializeOwned,
550
+ A: serde::Serialize + std::fmt::Debug,
551
+ {
552
+ trace!(
553
+ method,
554
+ client_index = self.config.client_index,
555
+ engine_count = self.engines.len(),
556
+ "sending utility request"
557
+ );
558
+
559
+ // Phase 1: allocate one call id per engine and build the per-engine
560
+ // request payloads up-front. Any failure here (registry closed, encode
561
+ // error) must roll back the call ids already allocated so they do not
562
+ // leak in the utility registry until shutdown.
563
+ let mut pending_calls = Vec::with_capacity(self.engines.len());
564
+ let mut prepared_sends = Vec::with_capacity(self.engines.len());
565
+ for engine in &self.engines {
566
+ let (call_id, rx) = match self.inner.allocate_and_register_utility_call() {
567
+ Ok(pair) => pair,
568
+ Err(err) => {
569
+ self.inner.unregister_utility_calls(pending_calls.iter().map(|(id, _)| *id));
570
+ return Err(err);
571
+ }
572
+ };
573
+ let request = match EngineCoreUtilityRequest::new(
574
+ self.config.client_index,
575
+ call_id,
576
+ method,
577
+ &args,
578
+ ) {
579
+ Ok(request) => request,
580
+ Err(err) => {
581
+ self.inner.unregister_utility_calls(
582
+ pending_calls.iter().map(|(id, _)| *id).chain(std::iter::once(call_id)),
583
+ );
584
+ return Err(err);
585
+ }
586
+ };
587
+ pending_calls.push((call_id, rx));
588
+ prepared_sends.push((&engine.engine_id, request));
589
+ }
590
+
591
+ // Phase 2: dispatch every utility request concurrently. `try_join_all`
592
+ // fails fast on the first transport error and drops the remaining send
593
+ // futures; any engines that already received the request will reply,
594
+ // but those replies are simply dropped because we roll back the call
595
+ // ids below.
596
+ let send_futures = prepared_sends.iter().map(|(engine_id, request)| {
597
+ self.inner.send_to_engine(engine_id, EngineCoreRequestType::Utility, request)
598
+ });
599
+ if let Err(err) = try_join_all(send_futures).await {
600
+ self.inner.unregister_utility_calls(pending_calls.iter().map(|(id, _)| *id));
601
+ return Err(err);
602
+ }
603
+
604
+ // Phase 3: wait for all engines to respond and preserve the per-engine
605
+ // result list.
606
+ let futures = pending_calls.into_iter().map(|(call_id, rx)| async move {
607
+ rx.await
608
+ .map_err(|_| Error::UtilityCallClosed {
609
+ method: method.to_string(),
610
+ call_id,
611
+ })??
612
+ .into_typed_result(method)
613
+ });
614
+ try_join_all(futures).await
615
+ }
616
+
617
+ /// Call a utility method on all connected engines and return the shared
618
+ /// result if every engine agrees.
619
+ pub async fn call_utility_consensus<T, A>(&self, method: &str, args: A) -> Result<T>
620
+ where
621
+ T: serde::de::DeserializeOwned + std::fmt::Debug + PartialEq,
622
+ A: serde::Serialize + std::fmt::Debug,
623
+ {
624
+ let results: Vec<T> = self.call_utility(method, args).await?;
625
+
626
+ if results.iter().all_equal() {
627
+ // `engine_count >= 1` is enforced during startup handshake so `results` must be
628
+ // non-empty.
629
+ Ok(results.into_iter().next().unwrap())
630
+ } else {
631
+ Err(Error::InconsistentUtilityResults {
632
+ method: method.to_string(),
633
+ values: format!("{results:?}"),
634
+ })
635
+ }
636
+ }
637
+
638
+ /// Execute `collective_rpc` on all engines and flatten all engine results
639
+ /// into one list.
640
+ pub async fn collective_rpc<A, K>(
641
+ &self,
642
+ method: &str,
643
+ timeout: Option<f64>,
644
+ args: A,
645
+ kwargs: K,
646
+ ) -> Result<Vec<rmpv::Value>>
647
+ where
648
+ A: serde::Serialize + std::fmt::Debug,
649
+ K: serde::Serialize + std::fmt::Debug,
650
+ {
651
+ let results = self
652
+ .call_utility::<rmpv::Value, _>("collective_rpc", (method, timeout, args, kwargs))
653
+ .await?;
654
+
655
+ Ok(results
656
+ .into_iter()
657
+ .flat_map(|result| match result {
658
+ // Each engine's `collective_rpc` result is itself the worker-level result list.
659
+ rmpv::Value::Array(results) => results,
660
+ other => vec![other],
661
+ })
662
+ .collect())
663
+ }
664
+
665
+ /// Return whether the engine is currently sleeping at any level.
666
+ pub async fn is_sleeping(&self) -> Result<bool> {
667
+ self.call_utility_consensus("is_sleeping", ()).await
668
+ }
669
+
670
+ /// Reset the multi-modal cache.
671
+ pub async fn reset_mm_cache(&self) -> Result<()> {
672
+ self.call_utility::<(), _>("reset_mm_cache", ()).await?;
673
+ Ok(())
674
+ }
675
+
676
+ /// Reset the encoder cache.
677
+ pub async fn reset_encoder_cache(&self) -> Result<()> {
678
+ self.call_utility::<(), _>("reset_encoder_cache", ()).await?;
679
+ Ok(())
680
+ }
681
+
682
+ /// Reset the prefix cache and optionally the external connector cache.
683
+ ///
684
+ /// Under data parallel, returns `true` only when every engine confirms the
685
+ /// reset (AND aggregation).
686
+ pub async fn reset_prefix_cache(
687
+ &self,
688
+ reset_running_requests: bool,
689
+ reset_connector: bool,
690
+ ) -> Result<bool> {
691
+ Ok(self
692
+ .call_utility(
693
+ "reset_prefix_cache",
694
+ (reset_running_requests, reset_connector),
695
+ )
696
+ .await?
697
+ .into_iter()
698
+ .all(|reset| reset))
699
+ }
700
+
701
+ /// Load or refresh one LoRA adapter on every connected engine.
702
+ pub async fn add_lora(&self, lora_request: &LoraRequest) -> Result<bool> {
703
+ Ok(self
704
+ .call_utility::<bool, _>("add_lora", (lora_request,))
705
+ .await?
706
+ .into_iter()
707
+ .all(|loaded| loaded))
708
+ }
709
+
710
+ /// Remove one LoRA adapter from every connected engine.
711
+ pub async fn remove_lora(&self, lora_id: u64) -> Result<bool> {
712
+ Ok(self
713
+ .call_utility::<bool, _>("remove_lora", (lora_id,))
714
+ .await?
715
+ .into_iter()
716
+ .all(|removed| removed))
717
+ }
718
+
719
+ /// Put the engine to sleep.
720
+ pub async fn sleep(&self, level: u32, mode: PauseMode) -> Result<()> {
721
+ self.call_utility::<(), _>("sleep", (level, mode)).await?;
722
+ Ok(())
723
+ }
724
+
725
+ /// Wake the engine from sleep, optionally limiting the wake-up to specific
726
+ /// tags.
727
+ pub async fn wake_up(&self, tags: Option<Vec<String>>) -> Result<()> {
728
+ self.call_utility::<(), _>("wake_up", (tags,)).await?;
729
+ Ok(())
730
+ }
731
+
732
+ /// Pause the scheduler so generation can be halted
733
+ pub async fn pause_scheduler(&self, mode: PauseMode, clear_cache: bool) -> Result<()> {
734
+ self.call_utility::<(), _>("pause_scheduler", (mode, clear_cache)).await?;
735
+ Ok(())
736
+ }
737
+
738
+ /// Resume the scheduler after a pause
739
+ pub async fn resume_scheduler(&self) -> Result<()> {
740
+ self.call_utility::<(), _>("resume_scheduler", ()).await?;
741
+ Ok(())
742
+ }
743
+
744
+ /// Return whether the scheduler is currently in any pause state.
745
+ pub async fn is_scheduler_paused(&self) -> Result<bool> {
746
+ self.call_utility_consensus("is_scheduler_paused", ()).await
747
+ }
748
+
749
+ /// Start profiling the engine.
750
+ pub async fn start_profile(&self, profile_prefix: Option<&str>) -> Result<()> {
751
+ self.call_utility::<(), _>("profile", (true, profile_prefix)).await?;
752
+ Ok(())
753
+ }
754
+
755
+ /// Stop profiling the engine.
756
+ pub async fn stop_profile(&self, profile_prefix: Option<&str>) -> Result<()> {
757
+ self.call_utility::<(), _>("profile", (false, profile_prefix)).await?;
758
+ Ok(())
759
+ }
760
+
761
+ /// Shut down local client tasks and close transport state.
762
+ pub async fn shutdown(self) -> Result<()> {
763
+ let Self {
764
+ inner,
765
+ abort_tx,
766
+ runtime,
767
+ output_task,
768
+ dispatcher_task,
769
+ abort_task,
770
+ coordinator_output_task,
771
+ coordinator_task,
772
+ ..
773
+ } = self;
774
+
775
+ info!("shutting down engine-core client");
776
+ inner.shutdown();
777
+ drop(abort_tx);
778
+
779
+ // Abort all client tasks first, then await them.
780
+ // Note the aborting orders here.
781
+ let mut tasks = vec![abort_task, dispatcher_task, output_task];
782
+ tasks.extend(coordinator_task);
783
+ tasks.extend(coordinator_output_task);
784
+
785
+ tasks.iter().for_each(|t| t.abort());
786
+ join_all(tasks).await;
787
+ drop(inner);
788
+ drop(runtime);
789
+
790
+ info!("engine-core client shut down");
791
+ Ok(())
792
+ }
793
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/client/imp.rs ADDED
@@ -0,0 +1,476 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::collections::{BTreeMap, BTreeSet};
2
+ use std::sync::Arc;
3
+ use std::time::{SystemTime, UNIX_EPOCH};
4
+
5
+ use arc_swap::ArcSwapOption;
6
+ use parking_lot::Mutex;
7
+ use thiserror_ext::AsReport as _;
8
+ use tokio::runtime::Handle;
9
+ use tokio::sync::mpsc;
10
+ use tracing::{debug, info, trace, warn};
11
+ use vllm_metrics::METRICS;
12
+ use zeromq::RouterSendHalf;
13
+
14
+ use crate::client::state::{OutputReceiver, RequestRegistry, UtilityReceiver, UtilityRegistry};
15
+ use crate::client::stream::EngineCoreStreamOutput;
16
+ use crate::client::{AbortCause, AbortRequest};
17
+ use crate::error::{client_closed, dispatcher_closed, unexpected_dispatcher_output};
18
+ use crate::metrics::{LoraInfoExporter, record_scheduler_stats};
19
+ use crate::protocol::encode_msgpack;
20
+ use crate::protocol::output::{EngineCoreOutput, EngineCoreOutputs};
21
+ use crate::protocol::request::EngineCoreRequestType;
22
+ use crate::protocol::stats::SchedulerStats;
23
+ use crate::protocol::utility::UtilityOutput;
24
+ use crate::transport::{ConnectedEngine, EngineId};
25
+ use crate::{Error, Result, transport};
26
+
27
+ pub(crate) struct ClientInner {
28
+ input_send: RouterSendHalf,
29
+ /// The runtime handle used for sending messages to the engine.
30
+ handle: Handle,
31
+ model_name: String,
32
+ request_reg: Mutex<RequestRegistry>,
33
+ utility_reg: Mutex<UtilityRegistry>,
34
+ health_error: ArcSwapOption<Error>,
35
+ }
36
+
37
+ impl ClientInner {
38
+ /// Create a new instance with the given input send half after the startup
39
+ /// handshake completes.
40
+ pub fn new(
41
+ input_send: RouterSendHalf,
42
+ handle: Handle,
43
+ model_name: String,
44
+ engines: &[ConnectedEngine],
45
+ ) -> Self {
46
+ Self {
47
+ input_send,
48
+ handle,
49
+ model_name,
50
+ request_reg: Mutex::new(RequestRegistry::new(engines)),
51
+ utility_reg: Mutex::new(UtilityRegistry::default()),
52
+ health_error: ArcSwapOption::empty(),
53
+ }
54
+ }
55
+
56
+ /// Get the model name associated with this client used for metrics
57
+ /// labeling.
58
+ pub fn model_name(&self) -> &str {
59
+ &self.model_name
60
+ }
61
+
62
+ /// Register a newly added request. Return the selected engine id and the
63
+ /// per-request output channel bound to its `request_id`.
64
+ ///
65
+ /// When `data_parallel_rank` is provided, the request is routed to that
66
+ /// specific engine rank, bypassing load balancing. `lora_name` is the
67
+ /// request's LoRA adapter, tracked for `vllm:lora_requests_info`.
68
+ pub fn register_request(
69
+ &self,
70
+ request_id: String,
71
+ lora_name: Option<String>,
72
+ data_parallel_rank: Option<u32>,
73
+ ) -> Result<(EngineId, OutputReceiver)> {
74
+ let mut registry = self.request_reg.lock();
75
+ if registry.is_closed() {
76
+ return Err(self.closed_error());
77
+ }
78
+ registry.register(request_id, lora_name, data_parallel_rank)
79
+ }
80
+
81
+ /// Allocate the next utility `call_id` and register its waiting receiver.
82
+ pub fn allocate_and_register_utility_call(&self) -> Result<(u64, UtilityReceiver)> {
83
+ let mut registry = self.utility_reg.lock();
84
+ if registry.is_closed() {
85
+ return Err(self.closed_error());
86
+ }
87
+ Ok(registry.allocate_and_register())
88
+ }
89
+
90
+ /// Undo a batch of utility call allocations when the fan-out send fails
91
+ /// partway through. Silently ignores unknown call ids so callers can pass
92
+ /// the full set without first filtering successful sends.
93
+ pub fn unregister_utility_calls(&self, call_ids: impl IntoIterator<Item = u64>) {
94
+ self.utility_reg.lock().unregister_many(call_ids);
95
+ }
96
+
97
+ /// Undo a request registration when `add_request()` fails.
98
+ pub fn rollback_request(&self, request_id: &str) {
99
+ let _ = self.request_reg.lock().remove(request_id);
100
+ }
101
+
102
+ /// Filter the given request IDs to the subset that are still tracked as
103
+ /// active and can be aborted, grouped by the engine that originally
104
+ /// accepted them.
105
+ pub fn abortable_request_ids(
106
+ &self,
107
+ request_ids: &[String],
108
+ ) -> Result<BTreeMap<EngineId, Vec<String>>> {
109
+ let registry = self.request_reg.lock();
110
+ if registry.is_closed() {
111
+ return Err(self.closed_error());
112
+ }
113
+ Ok(registry.abortable_request_ids(request_ids))
114
+ }
115
+
116
+ /// Obtain stream senders for a whole engine output batch with one registry
117
+ /// lock acquisition.
118
+ pub fn take_senders_for_outputs<'a>(
119
+ &self,
120
+ outputs: impl IntoIterator<Item = &'a EngineCoreOutput>,
121
+ ) -> Vec<Option<mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>>> {
122
+ self.request_reg.lock().senders_for_outputs(outputs)
123
+ }
124
+
125
+ /// Remove a batch of requests that have finished or aborted, returning
126
+ /// their stream senders.
127
+ pub fn finish_requests<'a>(
128
+ &self,
129
+ request_ids: impl IntoIterator<Item = &'a String>,
130
+ ) -> Vec<mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>> {
131
+ self.request_reg.lock().finish_many(request_ids)
132
+ }
133
+
134
+ /// Finalize client-initiated aborts by pushing a terminal `Abort` output
135
+ /// down each request's stream and removing it from the registry. Returns
136
+ /// the request ids that were still active. See [`RequestRegistry::abort_many`].
137
+ pub fn abort_requests_locally<'a>(
138
+ &self,
139
+ request_ids: impl IntoIterator<Item = &'a String>,
140
+ ) -> Vec<String> {
141
+ let timestamp = SystemTime::now()
142
+ .duration_since(UNIX_EPOCH)
143
+ .map(|d| d.as_secs_f64())
144
+ .unwrap_or(0.0);
145
+ self.request_reg.lock().abort_many(request_ids, timestamp)
146
+ }
147
+
148
+ /// Apply one scheduler stats update for the given engine to the local
149
+ /// routing state. Returns `false` if the engine is unknown to the
150
+ /// client.
151
+ pub fn apply_scheduler_stats(&self, engine_index: u32, stats: &SchedulerStats) -> bool {
152
+ self.request_reg.lock().apply_scheduler_stats(engine_index, stats)
153
+ }
154
+
155
+ /// Snapshot the adapter names of tracked LoRA requests as
156
+ /// (running, waiting) sets.
157
+ pub fn lora_adapter_states(&self) -> (BTreeSet<String>, BTreeSet<String>) {
158
+ self.request_reg.lock().lora_adapter_states()
159
+ }
160
+
161
+ /// Close all active request streams and utility calls with the first
162
+ /// persistent health error.
163
+ pub fn close_registries(&self, error: Arc<Error>) {
164
+ let persistent_error = self.record_health_error(error);
165
+ let request_senders = self.request_reg.lock().close();
166
+ let utility_senders = self.utility_reg.lock().close();
167
+
168
+ // Notify all ongoing requests that the client is closed.
169
+ for sender in request_senders {
170
+ let _ = sender.send(Err(Error::Shared(persistent_error.clone())));
171
+ }
172
+ for sender in utility_senders {
173
+ let _ = sender.send(Err(Error::Shared(persistent_error.clone())));
174
+ }
175
+ }
176
+
177
+ /// Return the first persistent health error observed by the client, if any.
178
+ pub fn health_error(&self) -> Option<Arc<Error>> {
179
+ self.health_error.load_full()
180
+ }
181
+
182
+ /// Return whether the client still considers the engine healthy.
183
+ pub fn is_healthy(&self) -> bool {
184
+ self.health_error.load().is_none()
185
+ }
186
+
187
+ /// Resolve one utility output to the waiting caller. Returns `true` if a
188
+ /// waiting caller existed.
189
+ pub fn resolve_utility_output(&self, output: UtilityOutput) -> bool {
190
+ let Some(call_id) = output.call_id.as_u64() else {
191
+ // Currently, all utility call issued by the client should have unsigned call IDs.
192
+ return false;
193
+ };
194
+
195
+ match self.utility_reg.lock().resolve(&call_id) {
196
+ Some(sender) => {
197
+ sender.send(Ok(output)).unwrap_or_default();
198
+ true
199
+ }
200
+ None => false,
201
+ }
202
+ }
203
+
204
+ /// Send the given message to the engine. The request should be first
205
+ /// registered via `register_request()` to ensure the request stream is
206
+ /// tracked.
207
+ pub async fn send_to_engine<T>(
208
+ &self,
209
+ engine_id: &EngineId,
210
+ request_type: EngineCoreRequestType,
211
+ payload: &T,
212
+ ) -> Result<()>
213
+ where
214
+ T: serde::Serialize + std::fmt::Debug,
215
+ {
216
+ // TODO: for `EngineCoreRequest`, split outbound tensor raw views into aux
217
+ // frames instead of always producing a single msgpack frame.
218
+ let payload = encode_msgpack(payload)?;
219
+ let mut input_send = self.input_send.clone();
220
+ let engine_id = engine_id.clone();
221
+
222
+ self.handle
223
+ .spawn(async move {
224
+ transport::send_message(
225
+ &mut input_send,
226
+ &engine_id,
227
+ request_type.to_frame(),
228
+ payload,
229
+ )
230
+ .await
231
+ })
232
+ .await?
233
+ }
234
+
235
+ /// Handle an abort request by sending the abort message to the engine.
236
+ pub async fn do_abort_requests(
237
+ &self,
238
+ engine_id: &EngineId,
239
+ request_ids: &[String],
240
+ ) -> Result<()> {
241
+ self.send_to_engine(engine_id, EngineCoreRequestType::Abort, &request_ids).await
242
+ }
243
+
244
+ /// Shut down by closing all active request streams and utility calls with a
245
+ /// sticky client closed error.
246
+ pub fn shutdown(&self) {
247
+ self.close_registries(Arc::new(client_closed!("engine-core client shut down")));
248
+ }
249
+
250
+ /// Remove the request from the active registry for auto-abort and return
251
+ /// the engine that the request was originally routed to, if it is still
252
+ /// active.
253
+ pub fn take_auto_abort_target(&self, request_id: &str) -> Option<EngineId> {
254
+ let mut registry = self.request_reg.lock();
255
+ let (_, engine_id) = registry.remove(request_id)?;
256
+ if registry.is_closed() {
257
+ return None;
258
+ }
259
+ Some(engine_id)
260
+ }
261
+
262
+ /// Publish the first persistent health error and return the sticky error
263
+ /// recorded for this client. Later failures do not overwrite the first
264
+ /// one so `/health` and post-close callers observe a stable cause.
265
+ fn record_health_error(&self, error: Arc<Error>) -> Arc<Error> {
266
+ if let Some(existing) = self.health_error.load_full() {
267
+ return existing;
268
+ }
269
+ self.health_error
270
+ .rcu(|current| current.clone().unwrap_or_else(|| error.clone()));
271
+ self.health_error
272
+ .load_full()
273
+ .expect("health error must be recorded before registries close")
274
+ }
275
+
276
+ /// Assert there is a recorded health error and return a `Shared` variant
277
+ /// wrapping it for error returns when the client is already closed.
278
+ fn closed_error(&self) -> Error {
279
+ Error::Shared(self.health_error.load_full().expect(
280
+ "closed registry must have a recorded health error before rejecting new operations",
281
+ ))
282
+ }
283
+ }
284
+
285
+ /// Background loop that listens for request IDs to abort and sends abort
286
+ /// messages to the engine. This is used to implement the auto-abort behavior
287
+ /// when a request stream is dropped without being properly terminated.
288
+ pub(crate) async fn run_abort_loop(
289
+ inner: Arc<ClientInner>,
290
+ mut abort_rx: mpsc::UnboundedReceiver<AbortRequest>,
291
+ ) {
292
+ // Coalesce bursts of auto-aborts into a single Abort message per engine.
293
+ // A dropped-stream storm (e.g. many clients disconnecting at once under
294
+ // high concurrency) would otherwise issue one engine round-trip per
295
+ // request. `recv_many` returns as soon as at least one item is ready, so a
296
+ // lone abort is still forwarded promptly.
297
+ const MAX_DRAIN: usize = 1024;
298
+ let mut batch: Vec<AbortRequest> = Vec::new();
299
+
300
+ while abort_rx.recv_many(&mut batch, MAX_DRAIN).await > 0 {
301
+ let mut by_engine: BTreeMap<EngineId, Vec<String>> = BTreeMap::new();
302
+
303
+ for AbortRequest { request_id, cause } in batch.drain(..) {
304
+ let Some(engine_id) = inner.take_auto_abort_target(&request_id) else {
305
+ debug!(request_id, "skip auto-abort for inactive request");
306
+ continue;
307
+ };
308
+
309
+ match cause {
310
+ AbortCause::DroppedStream => {
311
+ info!(request_id, "auto-aborting request due to dropped stream")
312
+ }
313
+ AbortCause::StopStringMatched => {
314
+ debug!(
315
+ request_id,
316
+ "auto-aborting request due to stop string matched"
317
+ )
318
+ }
319
+ }
320
+
321
+ by_engine.entry(engine_id).or_default().push(request_id);
322
+ }
323
+
324
+ for (engine_id, request_ids) in by_engine {
325
+ if let Err(error) = inner.do_abort_requests(&engine_id, &request_ids).await {
326
+ warn!(
327
+ ?engine_id,
328
+ ?request_ids,
329
+ error = %error.as_report(),
330
+ "failed to auto-abort request streams"
331
+ );
332
+ }
333
+ }
334
+ }
335
+ }
336
+
337
+ /// Background loop that listens for engine-core outputs and dispatches them to
338
+ /// the corresponding request streams based on their `request_id`.
339
+ pub(crate) async fn run_output_dispatcher_loop(
340
+ inner: Arc<ClientInner>,
341
+ mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
342
+ ) {
343
+ let mut lora_info = LoraInfoExporter::default();
344
+
345
+ let result: Result<()> = async {
346
+ loop {
347
+ let outputs = match output_rx.recv().await {
348
+ Some(outputs) => outputs,
349
+ None => Err(dispatcher_closed!(
350
+ "engine-core output dispatcher channel closed"
351
+ )),
352
+ }?;
353
+
354
+ match outputs {
355
+ EngineCoreOutputs::RequestBatch(batch) => {
356
+ let senders = inner.take_senders_for_outputs(&batch.outputs);
357
+ for (output, sender) in batch.outputs.into_iter().zip(senders) {
358
+ let request_id = output.request_id.clone();
359
+ let Some(sender) = sender else {
360
+ debug!(request_id, "dropping output for inactive request");
361
+ continue;
362
+ };
363
+
364
+ let wrapped_output = EngineCoreStreamOutput {
365
+ engine_index: batch.engine_index,
366
+ timestamp: batch.timestamp,
367
+ output,
368
+ };
369
+ if sender.send(Ok(wrapped_output)).is_err() {
370
+ debug!(request_id, "request output stream receiver dropped");
371
+ }
372
+ }
373
+
374
+ // The sender for normally-finished requests should have already been removed
375
+ // from the registry when their final output was dispatched
376
+ // above. This serves as a safety net to capture any
377
+ // requests marked as finished by the engine.
378
+ if let Some(finished_requests) = batch.finished_requests.as_ref() {
379
+ for request_id in finished_requests {
380
+ trace!(request_id, "request completed via finished_requests");
381
+ }
382
+ drop(inner.finish_requests(finished_requests));
383
+ }
384
+
385
+ if let Some(scheduler_stats) = batch.scheduler_stats.as_ref() {
386
+ if !inner.apply_scheduler_stats(batch.engine_index, scheduler_stats) {
387
+ debug!(
388
+ engine_index = batch.engine_index,
389
+ "dropping scheduler stats for unknown engine"
390
+ );
391
+ }
392
+ record_scheduler_stats(
393
+ &METRICS.scheduler,
394
+ inner.model_name(),
395
+ batch.engine_index,
396
+ scheduler_stats,
397
+ );
398
+ }
399
+
400
+ // The engine's scheduler stats never carry adapter names;
401
+ // the gauge is derived from the registry's frontend-side
402
+ // request tracking instead.
403
+ let (running, waiting) = inner.lora_adapter_states();
404
+ lora_info.update(&METRICS.scheduler, running, waiting);
405
+ }
406
+ EngineCoreOutputs::Utility(utility) => {
407
+ let call_id = utility.output.call_id;
408
+ if inner.resolve_utility_output(utility.output) {
409
+ trace!(
410
+ %call_id,
411
+ engine_index = utility.engine_index,
412
+ "resolved utility output"
413
+ );
414
+ } else {
415
+ warn!(
416
+ %call_id,
417
+ engine_index = utility.engine_index,
418
+ "dropping output for unexpected utility call"
419
+ );
420
+ }
421
+ }
422
+ other => {
423
+ Err::<(), _>(unexpected_dispatcher_output!(
424
+ "received unexpected output on main dispatcher path: {other:?}"
425
+ ))?;
426
+ }
427
+ }
428
+ }
429
+ }
430
+ .await;
431
+ let Err(error) = result else { return };
432
+
433
+ warn!(error = %error.as_report(), "output dispatcher exiting with error");
434
+ inner.close_registries(Arc::new(error));
435
+ }
436
+
437
+ #[cfg(test)]
438
+ mod tests {
439
+ use zeromq::{RouterSocket, Socket};
440
+
441
+ use super::*;
442
+ use crate::mock_engine::default_ready_response;
443
+
444
+ async fn test_inner() -> ClientInner {
445
+ let mut socket = RouterSocket::new();
446
+ socket.bind("tcp://127.0.0.1:0").await.unwrap();
447
+ let (send, _) = socket.split();
448
+ ClientInner::new(
449
+ send,
450
+ Handle::current(),
451
+ "test-model".to_string(),
452
+ &[ConnectedEngine {
453
+ engine_id: EngineId::from(b"engine-0"),
454
+ ready_response: default_ready_response(),
455
+ }],
456
+ )
457
+ }
458
+
459
+ #[tokio::test(flavor = "multi_thread", worker_threads = 2)]
460
+ async fn close_registries_records_first_health_error_only() {
461
+ let inner = test_inner().await;
462
+
463
+ inner.close_registries(Arc::new(Error::EngineCoreDead));
464
+ assert!(!inner.is_healthy());
465
+ assert!(matches!(
466
+ inner.health_error().as_deref(),
467
+ Some(Error::EngineCoreDead)
468
+ ));
469
+
470
+ inner.close_registries(Arc::new(client_closed!("shutdown")));
471
+ assert!(matches!(
472
+ inner.health_error().as_deref(),
473
+ Some(Error::EngineCoreDead)
474
+ ));
475
+ }
476
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/client/state.rs ADDED
@@ -0,0 +1,926 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::collections::{BTreeMap, BTreeSet, HashMap};
2
+ use std::sync::atomic::{AtomicU64, Ordering};
3
+
4
+ use tokio::sync::{mpsc, oneshot};
5
+ use tracing::trace;
6
+
7
+ use crate::EngineId;
8
+ use crate::client::stream::EngineCoreStreamOutput;
9
+ use crate::error::{Error, Result};
10
+ use crate::protocol::output::{EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput};
11
+ use crate::protocol::stats::SchedulerStats;
12
+ use crate::protocol::utility::UtilityOutput;
13
+ use crate::transport::ConnectedEngine;
14
+
15
+ pub type OutputSender = mpsc::UnboundedSender<Result<EngineCoreStreamOutput>>;
16
+ pub type OutputReceiver = mpsc::UnboundedReceiver<Result<EngineCoreStreamOutput>>;
17
+ pub type UtilitySender = oneshot::Sender<Result<UtilityOutput>>;
18
+ pub type UtilityReceiver = oneshot::Receiver<Result<UtilityOutput>>;
19
+
20
+ #[derive(Debug)]
21
+ struct TrackedRequest {
22
+ sender: OutputSender,
23
+ engine_id: EngineId,
24
+ lora: Option<LoraRequestState>,
25
+ }
26
+
27
+ /// Frontend-side view of one LoRA request's scheduling phase.
28
+ ///
29
+ /// The engine's `SchedulerStats` does not carry adapter names, so
30
+ /// `vllm:lora_requests_info` must be derived from per-request lifecycle events
31
+ /// observed by this client, mirroring `LoRARequestStates` in the Python
32
+ /// frontend (`vllm/v1/engine/output_processor.py`).
33
+ #[derive(Debug)]
34
+ struct LoraRequestState {
35
+ adapter_name: String,
36
+ phase: LoraPhase,
37
+ }
38
+
39
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
40
+ enum LoraPhase {
41
+ Waiting,
42
+ Running,
43
+ }
44
+
45
+ /// The latest real scheduler-side load snapshot observed from one engine.
46
+ ///
47
+ /// These counters come from `scheduler_stats` on the normal engine output path
48
+ /// and are the preferred routing signal once available.
49
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
50
+ struct EngineLoadSnapshot {
51
+ /// Requests still counted on the scheduler's waiting side.
52
+ waiting: usize,
53
+ /// Requests currently counted on the scheduler's running side.
54
+ running: usize,
55
+ }
56
+
57
+ #[derive(Debug, Default)]
58
+ struct EngineRoutingState {
59
+ /// Requests admitted by this frontend that have not finished yet.
60
+ ///
61
+ /// This is used both as the bootstrap fallback before real scheduler stats
62
+ /// exist and as a lower bound afterwards so asynchronous scheduler
63
+ /// snapshots cannot erase frontend admission history.
64
+ inflight: usize,
65
+ /// The latest real scheduler snapshot received from this engine, if any.
66
+ last_scheduler_stats: Option<EngineLoadSnapshot>,
67
+ }
68
+
69
+ impl EngineRoutingState {
70
+ /// Compute the routing score used to pick the least-loaded engine.
71
+ ///
72
+ /// Scheduler stats can raise the load estimate above the frontend-local
73
+ /// view, but they should not lower it below requests this frontend has
74
+ /// already admitted. Waiting requests still get the same extra penalty
75
+ /// as the original `waiting * 4 + running` score.
76
+ fn routing_score(&self) -> usize {
77
+ const WAITING_WEIGHT: usize = 4;
78
+
79
+ let Some(stats) = self.last_scheduler_stats else {
80
+ return self.inflight;
81
+ };
82
+
83
+ let scheduler_total = stats.running + stats.waiting;
84
+ self.inflight.max(scheduler_total) + stats.waiting * (WAITING_WEIGHT - 1)
85
+ }
86
+
87
+ /// Replace the local routing view with a fresh real scheduler snapshot.
88
+ fn apply_scheduler_counts(&mut self, next: EngineLoadSnapshot) {
89
+ self.last_scheduler_stats = Some(next);
90
+ }
91
+ }
92
+
93
+ /// Internal registry for tracking active requests and their output stream
94
+ /// senders.
95
+ ///
96
+ /// This is used to route incoming outputs to the correct request stream, and to
97
+ /// ensure proper cleanup of senders when requests finish or the client shuts
98
+ /// down.
99
+ #[derive(Debug)]
100
+ pub struct RequestRegistry {
101
+ closed: bool,
102
+ requests: HashMap<String, TrackedRequest>,
103
+ active_lora_requests: usize,
104
+ routing_per_engine: BTreeMap<EngineId, EngineRoutingState>,
105
+ }
106
+
107
+ impl RequestRegistry {
108
+ pub fn new(engines: &[ConnectedEngine]) -> Self {
109
+ Self {
110
+ closed: false,
111
+ requests: HashMap::default(),
112
+ active_lora_requests: 0,
113
+ routing_per_engine: engines
114
+ .iter()
115
+ .map(|engine| (engine.engine_id.clone(), EngineRoutingState::default()))
116
+ .collect(),
117
+ }
118
+ }
119
+
120
+ /// Register a newly added request. Create the per-request output channel
121
+ /// bound to its `request_id` and return the selected engine id.
122
+ ///
123
+ /// When `data_parallel_rank` is provided, the request is routed directly to
124
+ /// the engine at that rank index, bypassing load balancing. Otherwise
125
+ /// the engine with the fewest in-flight requests is chosen.
126
+ pub fn register(
127
+ &mut self,
128
+ request_id: String,
129
+ lora_name: Option<String>,
130
+ data_parallel_rank: Option<u32>,
131
+ ) -> Result<(EngineId, OutputReceiver)> {
132
+ if self.requests.contains_key(&request_id) {
133
+ return Err(Error::DuplicateRequestId { request_id });
134
+ }
135
+
136
+ let engine_id = self.choose_engine_for_request(data_parallel_rank)?;
137
+ let (tx, rx) = mpsc::unbounded_channel();
138
+ let lora = lora_name.map(|adapter_name| LoraRequestState {
139
+ adapter_name,
140
+ phase: LoraPhase::Waiting,
141
+ });
142
+ if lora.is_some() {
143
+ self.active_lora_requests += 1;
144
+ }
145
+ self.requests.insert(
146
+ request_id,
147
+ TrackedRequest {
148
+ sender: tx,
149
+ engine_id: engine_id.clone(),
150
+ lora,
151
+ },
152
+ );
153
+
154
+ let state = self
155
+ .routing_per_engine
156
+ .get_mut(&engine_id)
157
+ .expect("request registry must track all known engines");
158
+ state.inflight += 1;
159
+
160
+ Ok((engine_id, rx))
161
+ }
162
+
163
+ fn choose_engine_for_request(&mut self, data_parallel_rank: Option<u32>) -> Result<EngineId> {
164
+ if let Some(rank) = data_parallel_rank {
165
+ // Route to the engine at the specified rank index.
166
+ let engine_id = EngineId::from_engine_index(rank);
167
+ return self
168
+ .routing_per_engine
169
+ .contains_key(&engine_id)
170
+ .then_some(engine_id)
171
+ .ok_or_else(|| Error::InvalidDataParallelRank {
172
+ rank,
173
+ num_engines: self.routing_per_engine.len() as u32,
174
+ });
175
+ }
176
+
177
+ Ok(self
178
+ .routing_per_engine
179
+ .iter()
180
+ .min_by_key(|(_, state)| state.routing_score())
181
+ .map(|(engine_id, _)| engine_id.clone())
182
+ .expect("request registry must contain at least one engine"))
183
+ }
184
+
185
+ /// Filter the given request IDs to the subset that are still tracked as
186
+ /// active and can be aborted, grouped by engine.
187
+ pub fn abortable_request_ids(&self, request_ids: &[String]) -> BTreeMap<EngineId, Vec<String>> {
188
+ let mut by_engine = BTreeMap::new();
189
+ for request_id in request_ids {
190
+ let Some(tracked) = self.requests.get(request_id.as_str()) else {
191
+ continue;
192
+ };
193
+ by_engine
194
+ .entry(tracked.engine_id.clone())
195
+ .or_insert_with(Vec::new)
196
+ .push(request_id.clone());
197
+ }
198
+ by_engine
199
+ }
200
+
201
+ /// Obtain the stream sender for one output. If it indicates the request is
202
+ /// finished, it will be removed from the registry.
203
+ pub fn sender_for_output(&mut self, output: &EngineCoreOutput) -> Option<OutputSender> {
204
+ self.apply_lora_events(output);
205
+ if output.finished() {
206
+ self.remove(output.request_id.as_str()).map(|tracked| tracked.0)
207
+ } else {
208
+ self.requests
209
+ .get(output.request_id.as_str())
210
+ .map(|tracked| tracked.sender.clone())
211
+ }
212
+ }
213
+
214
+ /// Advance the request's LoRA scheduling phase from the engine-core events
215
+ /// attached to one output, mirroring the Python frontend's
216
+ /// `LoRARequestStates.update_from_events`.
217
+ fn apply_lora_events(&mut self, output: &EngineCoreOutput) {
218
+ let Some(events) = output.events.as_ref() else {
219
+ return;
220
+ };
221
+ let Some(lora) = self
222
+ .requests
223
+ .get_mut(output.request_id.as_str())
224
+ .and_then(|tracked| tracked.lora.as_mut())
225
+ else {
226
+ return;
227
+ };
228
+ for event in events {
229
+ lora.phase = match event.r#type {
230
+ EngineCoreEventType::Queued | EngineCoreEventType::Preempted => LoraPhase::Waiting,
231
+ EngineCoreEventType::Scheduled => LoraPhase::Running,
232
+ };
233
+ }
234
+ }
235
+
236
+ /// Snapshot the adapter names of tracked LoRA requests as
237
+ /// (running, waiting) sets. Feeds the `vllm:lora_requests_info` gauge.
238
+ pub fn lora_adapter_states(&self) -> (BTreeSet<String>, BTreeSet<String>) {
239
+ if self.active_lora_requests == 0 {
240
+ return (BTreeSet::new(), BTreeSet::new());
241
+ }
242
+
243
+ let mut running = BTreeSet::new();
244
+ let mut waiting = BTreeSet::new();
245
+ for lora in self.requests.values().filter_map(|tracked| tracked.lora.as_ref()) {
246
+ let set = match lora.phase {
247
+ LoraPhase::Running => &mut running,
248
+ LoraPhase::Waiting => &mut waiting,
249
+ };
250
+ set.insert(lora.adapter_name.clone());
251
+ }
252
+ (running, waiting)
253
+ }
254
+
255
+ /// Obtain stream senders for a whole engine output batch under one
256
+ /// registry lock. Finished outputs are removed before returning.
257
+ pub fn senders_for_outputs<'a>(
258
+ &mut self,
259
+ outputs: impl IntoIterator<Item = &'a EngineCoreOutput>,
260
+ ) -> Vec<Option<OutputSender>> {
261
+ outputs.into_iter().map(|output| self.sender_for_output(output)).collect()
262
+ }
263
+
264
+ /// Remove a batch of requests that have finished or aborted, returning
265
+ /// their stream senders.
266
+ pub fn finish_many<'a>(
267
+ &mut self,
268
+ request_ids: impl IntoIterator<Item = &'a String>,
269
+ ) -> Vec<OutputSender> {
270
+ request_ids
271
+ .into_iter()
272
+ .filter_map(|request_id| self.remove(request_id.as_str()).map(|tracked| tracked.0))
273
+ .collect()
274
+ }
275
+
276
+ /// Apply one scheduler stats update for the given engine to the local
277
+ /// routing state. Returns `false` if the engine is unknown to the
278
+ /// client.
279
+ pub fn apply_scheduler_stats(&mut self, engine_index: u32, stats: &SchedulerStats) -> bool {
280
+ self.apply_scheduler_counts(
281
+ engine_index,
282
+ EngineLoadSnapshot {
283
+ waiting: stats.num_waiting_reqs as usize,
284
+ running: stats.num_running_reqs as usize,
285
+ },
286
+ )
287
+ }
288
+
289
+ /// Mark the registry as closed, detach and return all tracked senders.
290
+ pub fn close(&mut self) -> Vec<OutputSender> {
291
+ if self.closed {
292
+ return Vec::new();
293
+ }
294
+
295
+ self.closed = true;
296
+ self.active_lora_requests = 0;
297
+ std::mem::take(&mut self.requests)
298
+ .into_values()
299
+ .map(|tracked| tracked.sender)
300
+ .collect()
301
+ }
302
+
303
+ /// Finalize client-initiated aborts: remove each request and push a
304
+ /// terminal output with `finish_reason = Abort` down its stream before the
305
+ /// sender drops. Returns the request ids that were still active.
306
+ pub fn abort_many<'a>(
307
+ &mut self,
308
+ request_ids: impl IntoIterator<Item = &'a String>,
309
+ timestamp: f64,
310
+ ) -> Vec<String> {
311
+ let mut aborted = Vec::new();
312
+ for request_id in request_ids {
313
+ let Some((sender, engine_id)) = self.remove(request_id) else {
314
+ continue;
315
+ };
316
+ let output = EngineCoreStreamOutput {
317
+ engine_index: engine_id.engine_index().unwrap_or(0),
318
+ timestamp,
319
+ output: EngineCoreOutput {
320
+ request_id: request_id.clone(),
321
+ finish_reason: Some(EngineCoreFinishReason::Abort),
322
+ ..EngineCoreOutput::default()
323
+ },
324
+ };
325
+ let _ = sender.send(Ok(output));
326
+ aborted.push(request_id.clone());
327
+ }
328
+ aborted
329
+ }
330
+
331
+ /// Remove one request from the local registry. Returns the tracked entry if
332
+ /// it exists.
333
+ #[must_use]
334
+ pub fn remove(&mut self, request_id: &str) -> Option<(OutputSender, EngineId)> {
335
+ let tracked = self.requests.remove(request_id)?;
336
+ if tracked.lora.is_some() {
337
+ self.active_lora_requests -= 1;
338
+ }
339
+ self.routing_per_engine
340
+ .get_mut(&tracked.engine_id)
341
+ .expect("request registry must track all known engines")
342
+ .inflight -= 1;
343
+ Some((tracked.sender, tracked.engine_id))
344
+ }
345
+
346
+ fn apply_scheduler_counts(&mut self, engine_index: u32, next: EngineLoadSnapshot) -> bool {
347
+ let engine_id = EngineId::from_engine_index(engine_index);
348
+ let Some(state) = self.routing_per_engine.get_mut(&engine_id) else {
349
+ return false;
350
+ };
351
+
352
+ let previous = state.last_scheduler_stats;
353
+ if previous != Some(next) {
354
+ trace!(
355
+ ?engine_id,
356
+ previous_waiting = previous.map(|stats| stats.waiting),
357
+ previous_running = previous.map(|stats| stats.running),
358
+ waiting = next.waiting,
359
+ running = next.running,
360
+ "updated scheduler routing counts",
361
+ );
362
+ }
363
+
364
+ state.apply_scheduler_counts(next);
365
+ true
366
+ }
367
+
368
+ #[cfg(test)]
369
+ pub fn contains(&self, request_id: &str) -> bool {
370
+ self.requests.contains_key(request_id)
371
+ }
372
+
373
+ pub fn is_closed(&self) -> bool {
374
+ self.closed
375
+ }
376
+
377
+ #[cfg(test)]
378
+ fn active_lora_requests(&self) -> usize {
379
+ self.active_lora_requests
380
+ }
381
+ }
382
+
383
+ /// Internal registry for tracking active utility calls and their waiting
384
+ /// receivers.
385
+ #[derive(Debug)]
386
+ pub struct UtilityRegistry {
387
+ closed: bool,
388
+ next_call_id: AtomicU64,
389
+ utility_calls: BTreeMap<u64, UtilitySender>,
390
+ }
391
+
392
+ impl Default for UtilityRegistry {
393
+ fn default() -> Self {
394
+ Self {
395
+ closed: false,
396
+ next_call_id: AtomicU64::new(1),
397
+ utility_calls: BTreeMap::default(),
398
+ }
399
+ }
400
+ }
401
+
402
+ impl UtilityRegistry {
403
+ /// Allocate the next utility `call_id` and register a newly added utility
404
+ /// call.
405
+ pub fn allocate_and_register(&mut self) -> (u64, UtilityReceiver) {
406
+ let call_id = self.next_call_id.fetch_add(1, Ordering::Relaxed);
407
+ let (tx, rx) = oneshot::channel();
408
+ self.utility_calls.insert(call_id, tx);
409
+ (call_id, rx)
410
+ }
411
+
412
+ /// Resolve a utility output to its waiting receiver.
413
+ pub fn resolve(&mut self, call_id: &u64) -> Option<UtilitySender> {
414
+ self.utility_calls.remove(call_id)
415
+ }
416
+
417
+ /// Drop a batch of registered utility calls without delivering a result.
418
+ /// Used to roll back allocations when the dispatch fan-out fails before
419
+ /// every engine could accept the request.
420
+ pub fn unregister_many(&mut self, call_ids: impl IntoIterator<Item = u64>) {
421
+ for call_id in call_ids {
422
+ self.utility_calls.remove(&call_id);
423
+ }
424
+ }
425
+
426
+ /// Mark the registry as closed, detach and return all tracked senders.
427
+ pub fn close(&mut self) -> Vec<UtilitySender> {
428
+ if self.closed {
429
+ return Vec::new();
430
+ }
431
+
432
+ self.closed = true;
433
+ std::mem::take(&mut self.utility_calls).into_values().collect()
434
+ }
435
+
436
+ #[cfg(test)]
437
+ pub fn contains(&self, call_id: u64) -> bool {
438
+ self.utility_calls.contains_key(&call_id)
439
+ }
440
+
441
+ pub fn is_closed(&self) -> bool {
442
+ self.closed
443
+ }
444
+ }
445
+
446
+ #[cfg(test)]
447
+ mod tests {
448
+ use std::collections::BTreeSet;
449
+
450
+ use crate::EngineId;
451
+ use crate::client::state::{
452
+ EngineLoadSnapshot, EngineRoutingState, RequestRegistry, UtilityRegistry,
453
+ };
454
+ use crate::mock_engine::default_ready_response;
455
+ use crate::protocol::output::{
456
+ EngineCoreEvent, EngineCoreEventType, EngineCoreFinishReason, EngineCoreOutput,
457
+ };
458
+ use crate::transport::ConnectedEngine;
459
+
460
+ fn connected_engine(engine_id: EngineId) -> ConnectedEngine {
461
+ ConnectedEngine {
462
+ engine_id,
463
+ ready_response: default_ready_response(),
464
+ }
465
+ }
466
+
467
+ fn output_with_events(
468
+ request_id: &str,
469
+ events: &[EngineCoreEventType],
470
+ finish_reason: Option<EngineCoreFinishReason>,
471
+ ) -> EngineCoreOutput {
472
+ EngineCoreOutput {
473
+ request_id: request_id.to_string(),
474
+ events: Some(
475
+ events
476
+ .iter()
477
+ .map(|event_type| EngineCoreEvent {
478
+ r#type: *event_type,
479
+ timestamp: 0.0,
480
+ })
481
+ .collect(),
482
+ ),
483
+ finish_reason,
484
+ ..Default::default()
485
+ }
486
+ }
487
+
488
+ fn adapter_names(values: &[&str]) -> BTreeSet<String> {
489
+ values.iter().map(|name| (*name).to_string()).collect()
490
+ }
491
+
492
+ #[test]
493
+ fn registry_rejects_duplicate_request_ids() {
494
+ let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
495
+ registry.register("req-1".to_string(), None, None).unwrap();
496
+ let error = registry.register("req-1".to_string(), None, None).unwrap_err();
497
+ assert!(matches!(
498
+ error,
499
+ crate::error::Error::DuplicateRequestId { request_id } if request_id == "req-1"
500
+ ));
501
+ }
502
+
503
+ #[test]
504
+ fn registry_removes_finished_request_on_output() {
505
+ let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
506
+ registry.register("req-1".to_string(), None, None).unwrap();
507
+
508
+ let sender = registry.sender_for_output(&EngineCoreOutput {
509
+ request_id: "req-1".to_string(),
510
+ finish_reason: Some(EngineCoreFinishReason::Length),
511
+ ..Default::default()
512
+ });
513
+
514
+ assert!(sender.is_some());
515
+ assert!(!registry.contains("req-1"));
516
+ }
517
+
518
+ #[test]
519
+ fn registry_tracks_lora_phases_from_engine_events() {
520
+ let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
521
+ registry
522
+ .register("req-lora".to_string(), Some("adapter-a".to_string()), None)
523
+ .unwrap();
524
+ registry.register("req-plain".to_string(), None, None).unwrap();
525
+
526
+ // Registered but not yet scheduled: counted as waiting. The non-LoRA
527
+ // request never shows up.
528
+ assert_eq!(
529
+ registry.lora_adapter_states(),
530
+ (adapter_names(&[]), adapter_names(&["adapter-a"]))
531
+ );
532
+
533
+ // Queued then scheduled in one output: running.
534
+ drop(registry.sender_for_output(&output_with_events(
535
+ "req-lora",
536
+ &[EngineCoreEventType::Queued, EngineCoreEventType::Scheduled],
537
+ None,
538
+ )));
539
+ assert_eq!(
540
+ registry.lora_adapter_states(),
541
+ (adapter_names(&["adapter-a"]), adapter_names(&[]))
542
+ );
543
+
544
+ // Preempted: back to waiting.
545
+ drop(registry.sender_for_output(&output_with_events(
546
+ "req-lora",
547
+ &[EngineCoreEventType::Preempted],
548
+ None,
549
+ )));
550
+ assert_eq!(
551
+ registry.lora_adapter_states(),
552
+ (adapter_names(&[]), adapter_names(&["adapter-a"]))
553
+ );
554
+
555
+ // Finished: dropped from tracking entirely.
556
+ drop(registry.sender_for_output(&output_with_events(
557
+ "req-lora",
558
+ &[EngineCoreEventType::Scheduled],
559
+ Some(EngineCoreFinishReason::Stop),
560
+ )));
561
+ assert_eq!(
562
+ registry.lora_adapter_states(),
563
+ (adapter_names(&[]), adapter_names(&[]))
564
+ );
565
+ }
566
+
567
+ #[test]
568
+ fn registry_unions_lora_adapters_across_requests() {
569
+ let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
570
+ registry
571
+ .register("req-a1".to_string(), Some("adapter-a".to_string()), None)
572
+ .unwrap();
573
+ registry
574
+ .register("req-a2".to_string(), Some("adapter-a".to_string()), None)
575
+ .unwrap();
576
+ registry
577
+ .register("req-b".to_string(), Some("adapter-b".to_string()), None)
578
+ .unwrap();
579
+
580
+ // One of adapter-a's requests starts running while the other waits:
581
+ // the adapter appears in both sets.
582
+ drop(registry.sender_for_output(&output_with_events(
583
+ "req-a1",
584
+ &[EngineCoreEventType::Scheduled],
585
+ None,
586
+ )));
587
+ assert_eq!(
588
+ registry.lora_adapter_states(),
589
+ (
590
+ adapter_names(&["adapter-a"]),
591
+ adapter_names(&["adapter-a", "adapter-b"])
592
+ )
593
+ );
594
+ }
595
+
596
+ #[test]
597
+ fn registry_counts_only_active_lora_requests() {
598
+ let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
599
+
600
+ registry.register("req-plain".to_string(), None, None).unwrap();
601
+ assert_eq!(registry.active_lora_requests(), 0);
602
+ assert_eq!(
603
+ registry.lora_adapter_states(),
604
+ (adapter_names(&[]), adapter_names(&[]))
605
+ );
606
+
607
+ registry
608
+ .register(
609
+ "req-lora-a".to_string(),
610
+ Some("adapter-a".to_string()),
611
+ None,
612
+ )
613
+ .unwrap();
614
+ registry
615
+ .register(
616
+ "req-lora-b".to_string(),
617
+ Some("adapter-b".to_string()),
618
+ None,
619
+ )
620
+ .unwrap();
621
+ assert_eq!(registry.active_lora_requests(), 2);
622
+
623
+ drop(registry.remove("req-plain"));
624
+ assert_eq!(registry.active_lora_requests(), 2);
625
+
626
+ drop(registry.finish_many(&["req-lora-a".to_string()]));
627
+ assert_eq!(registry.active_lora_requests(), 1);
628
+
629
+ drop(registry.abort_many(&["req-lora-b".to_string()], 0.0));
630
+ assert_eq!(registry.active_lora_requests(), 0);
631
+ assert_eq!(
632
+ registry.lora_adapter_states(),
633
+ (adapter_names(&[]), adapter_names(&[]))
634
+ );
635
+ }
636
+
637
+ #[test]
638
+ fn registry_clears_lora_count_on_close() {
639
+ let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
640
+ registry
641
+ .register("req-lora".to_string(), Some("adapter-a".to_string()), None)
642
+ .unwrap();
643
+
644
+ assert_eq!(registry.active_lora_requests(), 1);
645
+ drop(registry.close());
646
+ assert_eq!(registry.active_lora_requests(), 0);
647
+ assert_eq!(
648
+ registry.lora_adapter_states(),
649
+ (adapter_names(&[]), adapter_names(&[]))
650
+ );
651
+ }
652
+
653
+ #[test]
654
+ fn registry_drops_lora_tracking_on_abort() {
655
+ let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
656
+ registry
657
+ .register("req-lora".to_string(), Some("adapter-a".to_string()), None)
658
+ .unwrap();
659
+
660
+ drop(registry.finish_many(&["req-lora".to_string()]));
661
+
662
+ assert_eq!(
663
+ registry.lora_adapter_states(),
664
+ (adapter_names(&[]), adapter_names(&[]))
665
+ );
666
+ }
667
+
668
+ #[test]
669
+ fn registry_closes_all_requests_on_failure() {
670
+ let mut registry = RequestRegistry::new(&[connected_engine(EngineId::from(b"engine-0"))]);
671
+ registry.register("req-1".to_string(), None, None).unwrap();
672
+ registry.register("req-2".to_string(), None, None).unwrap();
673
+
674
+ let senders = registry.close();
675
+
676
+ assert_eq!(senders.len(), 2);
677
+ assert!(registry.is_closed());
678
+ }
679
+
680
+ #[test]
681
+ fn registry_tracks_engine_id_per_request() {
682
+ let engine_0 = EngineId::from_engine_index(0);
683
+ let engine_1 = EngineId::from_engine_index(1);
684
+ let mut registry = RequestRegistry::new(&[
685
+ connected_engine(engine_0.clone()),
686
+ connected_engine(engine_1.clone()),
687
+ ]);
688
+ let (chosen_0, _) = registry.register("req-1".to_string(), None, None).unwrap();
689
+ let (chosen_1, _) = registry.register("req-2".to_string(), None, None).unwrap();
690
+ let (chosen_0_again, _) = registry.register("req-3".to_string(), None, None).unwrap();
691
+
692
+ assert_eq!(chosen_0, engine_0);
693
+ assert_eq!(chosen_1, engine_1);
694
+ assert_eq!(chosen_0_again, engine_0);
695
+
696
+ let grouped = registry.abortable_request_ids(&[
697
+ "req-1".to_string(),
698
+ "req-2".to_string(),
699
+ "req-3".to_string(),
700
+ ]);
701
+ assert_eq!(
702
+ grouped.get(&engine_0).unwrap(),
703
+ &vec!["req-1".to_string(), "req-3".to_string()]
704
+ );
705
+ assert_eq!(grouped.get(&engine_1).unwrap(), &vec!["req-2".to_string()]);
706
+ }
707
+
708
+ #[test]
709
+ fn registry_uses_inflight_as_waiting_fallback_before_stats_arrive() {
710
+ let engine_0 = EngineId::from_engine_index(0);
711
+ let engine_1 = EngineId::from_engine_index(1);
712
+ let mut registry = RequestRegistry::new(&[
713
+ connected_engine(engine_0.clone()),
714
+ connected_engine(engine_1.clone()),
715
+ ]);
716
+
717
+ let (chosen_0, _) = registry.register("req-1".to_string(), None, None).unwrap();
718
+ let (chosen_1, _) = registry.register("req-2".to_string(), None, None).unwrap();
719
+ let (chosen_0_again, _) = registry.register("req-3".to_string(), None, None).unwrap();
720
+
721
+ assert_eq!(chosen_0, engine_0);
722
+ assert_eq!(chosen_1, engine_1);
723
+ assert_eq!(chosen_0_again, engine_0);
724
+ }
725
+
726
+ #[test]
727
+ fn routing_score_uses_inflight_before_stats_arrive() {
728
+ let state = EngineRoutingState {
729
+ inflight: 3,
730
+ last_scheduler_stats: None,
731
+ };
732
+
733
+ assert_eq!(state.routing_score(), 3);
734
+ }
735
+
736
+ #[test]
737
+ fn routing_score_uses_inflight_as_scheduler_stats_lower_bound() {
738
+ let state = EngineRoutingState {
739
+ inflight: 7,
740
+ last_scheduler_stats: Some(EngineLoadSnapshot {
741
+ waiting: 0,
742
+ running: 2,
743
+ }),
744
+ };
745
+
746
+ assert_eq!(state.routing_score(), 7);
747
+ }
748
+
749
+ #[test]
750
+ fn routing_score_keeps_extra_waiting_penalty() {
751
+ let state = EngineRoutingState {
752
+ inflight: 1,
753
+ last_scheduler_stats: Some(EngineLoadSnapshot {
754
+ waiting: 3,
755
+ running: 2,
756
+ }),
757
+ };
758
+
759
+ assert_eq!(state.routing_score(), 14);
760
+ }
761
+
762
+ #[test]
763
+ fn registry_prefers_real_scheduler_stats_over_inflight() {
764
+ let engine_0 = EngineId::from_engine_index(0);
765
+ let engine_1 = EngineId::from_engine_index(1);
766
+ let mut registry = RequestRegistry::new(&[
767
+ connected_engine(engine_0.clone()),
768
+ connected_engine(engine_1.clone()),
769
+ ]);
770
+
771
+ assert!(registry.apply_scheduler_counts(
772
+ 0,
773
+ EngineLoadSnapshot {
774
+ waiting: 3,
775
+ running: 2
776
+ }
777
+ ));
778
+ assert!(registry.apply_scheduler_counts(
779
+ 1,
780
+ EngineLoadSnapshot {
781
+ waiting: 0,
782
+ running: 1
783
+ }
784
+ ));
785
+
786
+ let (chosen, _) = registry.register("req-stats".to_string(), None, None).unwrap();
787
+ assert_eq!(chosen, engine_1);
788
+ }
789
+
790
+ #[test]
791
+ fn register_with_data_parallel_rank_routes_to_specified_engine() {
792
+ let engine_0 = EngineId::from_engine_index(0);
793
+ let engine_1 = EngineId::from_engine_index(1);
794
+ let engine_2 = EngineId::from_engine_index(2);
795
+ let mut registry = RequestRegistry::new(&[
796
+ connected_engine(engine_0.clone()),
797
+ connected_engine(engine_1.clone()),
798
+ connected_engine(engine_2.clone()),
799
+ ]);
800
+
801
+ // Explicitly target rank 2 (third engine).
802
+ let (chosen, _) = registry.register("req-1".to_string(), None, Some(2)).unwrap();
803
+ assert_eq!(chosen, engine_2);
804
+
805
+ // Explicitly target rank 0 (first engine).
806
+ let (chosen, _) = registry.register("req-2".to_string(), None, Some(0)).unwrap();
807
+ assert_eq!(chosen, engine_0);
808
+
809
+ // Explicitly target rank 1.
810
+ let (chosen, _) = registry.register("req-3".to_string(), None, Some(1)).unwrap();
811
+ assert_eq!(chosen, engine_1);
812
+ }
813
+
814
+ #[test]
815
+ fn register_with_data_parallel_rank_bypasses_load_balancing() {
816
+ let engine_0 = EngineId::from_engine_index(0);
817
+ let engine_1 = EngineId::from_engine_index(1);
818
+ let mut registry = RequestRegistry::new(&[
819
+ connected_engine(engine_0.clone()),
820
+ connected_engine(engine_1.clone()),
821
+ ]);
822
+
823
+ // Load-balance: first two go to engine_0 and engine_1.
824
+ registry.register("req-lb-0".to_string(), None, None).unwrap();
825
+
826
+ // Now engine_0 has 1 in-flight. Without dp_rank, next would go to engine_1.
827
+ // But with dp_rank=0, it should still go to engine_0.
828
+ let (chosen, _) = registry.register("req-dp".to_string(), None, Some(0)).unwrap();
829
+ assert_eq!(chosen, engine_0);
830
+ }
831
+
832
+ #[test]
833
+ fn register_with_out_of_range_rank_returns_error() {
834
+ let mut registry = RequestRegistry::new(&[
835
+ connected_engine(EngineId::from_engine_index(0)),
836
+ connected_engine(EngineId::from_engine_index(1)),
837
+ ]);
838
+
839
+ let error = registry.register("req-1".to_string(), None, Some(2)).unwrap_err();
840
+ assert!(matches!(
841
+ error,
842
+ crate::error::Error::InvalidDataParallelRank {
843
+ rank: 2,
844
+ num_engines: 2,
845
+ }
846
+ ));
847
+ }
848
+
849
+ #[test]
850
+ fn register_with_rank_on_single_engine_only_accepts_zero() {
851
+ let engine_0 = EngineId::from_engine_index(0);
852
+ let mut registry = RequestRegistry::new(&[connected_engine(engine_0.clone())]);
853
+
854
+ let (chosen, _) = registry.register("req-ok".to_string(), None, Some(0)).unwrap();
855
+ assert_eq!(chosen, engine_0);
856
+
857
+ let error = registry.register("req-bad".to_string(), None, Some(1)).unwrap_err();
858
+ assert!(matches!(
859
+ error,
860
+ crate::error::Error::InvalidDataParallelRank {
861
+ rank: 1,
862
+ num_engines: 1,
863
+ }
864
+ ));
865
+ }
866
+
867
+ #[test]
868
+ fn utility_registry_tracks_and_removes_call_ids() {
869
+ let mut registry = UtilityRegistry::default();
870
+ let (call_id_1, _) = registry.allocate_and_register();
871
+ let (call_id_2, _) = registry.allocate_and_register();
872
+
873
+ assert_eq!(call_id_1, 1);
874
+ assert_eq!(call_id_2, 2);
875
+ assert!(registry.contains(1));
876
+ assert!(registry.contains(2));
877
+ assert!(registry.resolve(&1).is_some());
878
+ assert!(!registry.contains(1));
879
+ assert!(registry.contains(2));
880
+ }
881
+
882
+ #[test]
883
+ fn utility_registry_closes_all_waiters_on_failure() {
884
+ let mut registry = UtilityRegistry::default();
885
+ registry.allocate_and_register();
886
+ registry.allocate_and_register();
887
+
888
+ let senders = registry.close();
889
+
890
+ assert_eq!(senders.len(), 2);
891
+ assert!(!registry.contains(1));
892
+ assert!(!registry.contains(2));
893
+ assert!(registry.is_closed());
894
+ }
895
+
896
+ #[test]
897
+ fn utility_registry_unregister_many_drops_pending_calls() {
898
+ use tokio::sync::oneshot::error::TryRecvError;
899
+
900
+ let mut registry = UtilityRegistry::default();
901
+ let (call_id_1, mut rx_1) = registry.allocate_and_register();
902
+ let (call_id_2, mut rx_2) = registry.allocate_and_register();
903
+ let (call_id_3, _rx_3) = registry.allocate_and_register();
904
+
905
+ // Drop two of the three allocated calls; the third stays pending.
906
+ registry.unregister_many([call_id_1, call_id_2]);
907
+
908
+ assert!(!registry.contains(call_id_1));
909
+ assert!(!registry.contains(call_id_2));
910
+ assert!(registry.contains(call_id_3));
911
+ // The receivers must observe the sender being dropped (channel closed).
912
+ assert!(matches!(rx_1.try_recv(), Err(TryRecvError::Closed)));
913
+ assert!(matches!(rx_2.try_recv(), Err(TryRecvError::Closed)));
914
+ }
915
+
916
+ #[test]
917
+ fn utility_registry_unregister_many_ignores_unknown_call_ids() {
918
+ let mut registry = UtilityRegistry::default();
919
+ let (call_id, _rx) = registry.allocate_and_register();
920
+
921
+ // Unknown call ids are silently ignored — caller doesn't care which were live.
922
+ registry.unregister_many([call_id, 42, 9999]);
923
+
924
+ assert!(!registry.contains(call_id));
925
+ }
926
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/client/stream.rs ADDED
@@ -0,0 +1,153 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::ops::Deref;
2
+ use std::pin::Pin;
3
+ use std::task::{Context, Poll};
4
+
5
+ use futures::Stream;
6
+ use futures::stream::FusedStream;
7
+ use thiserror_ext::AsReport as _;
8
+ use tokio::sync::mpsc;
9
+ use tracing::{debug, error, warn};
10
+
11
+ use crate::client::AbortRequest;
12
+ use crate::client::state::OutputReceiver;
13
+ use crate::protocol::output::{EngineCoreFinishReason, EngineCoreOutput};
14
+ use crate::{AbortCause, Error, Result};
15
+
16
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
17
+ enum State {
18
+ Running,
19
+ Finished,
20
+ ClosedWithError,
21
+ UnexpectedClose,
22
+ }
23
+
24
+ /// One request-scoped engine-core output plus the enclosing batch metadata.
25
+ #[derive(Debug, Clone, PartialEq)]
26
+ pub struct EngineCoreStreamOutput {
27
+ pub engine_index: u32,
28
+ pub timestamp: f64,
29
+ pub output: EngineCoreOutput,
30
+ }
31
+
32
+ impl Deref for EngineCoreStreamOutput {
33
+ type Target = EngineCoreOutput;
34
+
35
+ fn deref(&self) -> &Self::Target {
36
+ &self.output
37
+ }
38
+ }
39
+
40
+ /// Stream of raw engine-core outputs for one request.
41
+ ///
42
+ /// The stream yields only [`EngineCoreStreamOutput`] values whose embedded
43
+ /// output `request_id` matches the originating `add_request()` call. Normal
44
+ /// request completion is expected to include a final output object whose
45
+ /// `finish_reason` is non-`None`.
46
+ pub struct EngineCoreOutputStream {
47
+ request_id: String,
48
+ abort_tx: mpsc::UnboundedSender<AbortRequest>,
49
+ state: State,
50
+ rx: OutputReceiver,
51
+ }
52
+
53
+ impl EngineCoreOutputStream {
54
+ pub(crate) fn new(
55
+ request_id: String,
56
+ abort_tx: mpsc::UnboundedSender<AbortRequest>,
57
+ rx: OutputReceiver,
58
+ ) -> Self {
59
+ Self {
60
+ request_id,
61
+ abort_tx,
62
+ state: State::Running,
63
+ rx,
64
+ }
65
+ }
66
+
67
+ /// Return the engine-core `request_id` bound to this stream.
68
+ pub fn request_id(&self) -> &str {
69
+ &self.request_id
70
+ }
71
+ }
72
+
73
+ impl Stream for EngineCoreOutputStream {
74
+ type Item = Result<EngineCoreStreamOutput>;
75
+
76
+ fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
77
+ if self.is_terminated() {
78
+ return Poll::Ready(None);
79
+ }
80
+
81
+ match Pin::new(&mut self.rx).poll_recv(cx) {
82
+ Poll::Pending => Poll::Pending,
83
+ Poll::Ready(Some(item)) => {
84
+ match &item {
85
+ Ok(output) => {
86
+ // If the output indicates the request is finished, mark the stream as
87
+ // terminated with cleanly-finished state and expect no more outputs to
88
+ // come.
89
+ if output.finished() {
90
+ if output.finish_reason == Some(EngineCoreFinishReason::Error) {
91
+ error!(
92
+ self.request_id,
93
+ "request failed with an internal error during generation"
94
+ );
95
+ }
96
+ debug!(self.request_id, "request completed via final output");
97
+ self.state = State::Finished;
98
+ }
99
+ }
100
+ Err(error) => {
101
+ // If we get an error from the output stream, mark the stream as terminated
102
+ // with an error.
103
+ warn!(self.request_id, error = %error.as_report(), "request encountered an error");
104
+ self.state = State::ClosedWithError;
105
+ }
106
+ }
107
+ Poll::Ready(Some(item))
108
+ }
109
+ Poll::Ready(None) => {
110
+ // If we get a `None` without seeing a finished output, this is an unexpected
111
+ // close from the engine side. Mark the stream as terminated
112
+ // with an unexpected close state and send an error down the
113
+ // stream to notify the caller.
114
+ warn!(self.request_id, "request stream closed unexpectedly");
115
+ self.state = State::UnexpectedClose;
116
+
117
+ Poll::Ready(Some(Err(Error::RequestStreamClosed {
118
+ request_id: self.request_id.clone(),
119
+ })))
120
+ }
121
+ }
122
+ }
123
+ }
124
+
125
+ impl FusedStream for EngineCoreOutputStream {
126
+ fn is_terminated(&self) -> bool {
127
+ !matches!(self.state, State::Running)
128
+ }
129
+ }
130
+
131
+ impl Drop for EngineCoreOutputStream {
132
+ fn drop(&mut self) {
133
+ if self.is_terminated() {
134
+ // If it's terminated, it means that the request either finished cleanly, or
135
+ // encountered an error or unexpected close from the engine. In any
136
+ // case, the request stream is already considered inactive and
137
+ // there's no need to abort it on the engine side.
138
+ return;
139
+ }
140
+
141
+ let abort_req = AbortRequest {
142
+ request_id: self.request_id.clone(),
143
+ cause: AbortCause::current(),
144
+ };
145
+
146
+ if self.abort_tx.send(abort_req).is_err() {
147
+ warn!(
148
+ request_id = self.request_id,
149
+ "auto-abort worker already shut down; skip auto-abort"
150
+ );
151
+ }
152
+ }
153
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/coordinator/bootstrap.rs ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::time::Duration;
2
+
3
+ use bytes::Bytes;
4
+ use zeromq::prelude::{Socket, SocketRecv, SocketSend};
5
+ use zeromq::{PullSocket, XPubSocket, ZmqMessage};
6
+
7
+ use crate::error::{Error, Result, bail_unexpected_handshake_message};
8
+
9
+ /// Engine-facing sockets owned by the in-process coordinator.
10
+ pub(crate) struct CoordinatorBootstrap {
11
+ pub input_address: String,
12
+ pub output_address: String,
13
+ pub input_socket: XPubSocket,
14
+ pub output_socket: PullSocket,
15
+ }
16
+
17
+ impl CoordinatorBootstrap {
18
+ /// Bind the engine-facing coordinator sockets on the given host.
19
+ pub(crate) async fn bind(local_host: &str) -> Result<Self> {
20
+ let mut input_socket = XPubSocket::new();
21
+ let input_address = input_socket.bind(&format!("tcp://{local_host}:0")).await?.to_string();
22
+
23
+ let mut output_socket = PullSocket::new();
24
+ let output_address =
25
+ output_socket.bind(&format!("tcp://{local_host}:0")).await?.to_string();
26
+
27
+ Ok(Self {
28
+ input_address,
29
+ output_address,
30
+ input_socket,
31
+ output_socket,
32
+ })
33
+ }
34
+
35
+ /// Complete the engine-facing startup gate before engines are allowed to
36
+ /// send handshake READY.
37
+ pub(crate) async fn wait_for_startup_gate(
38
+ &mut self,
39
+ engine_count: usize,
40
+ ready_timeout: Duration,
41
+ ) -> Result<()> {
42
+ wait_for_engine_subscriptions(&mut self.input_socket, engine_count, ready_timeout).await?;
43
+ send_ready_to_engines(&mut self.input_socket).await?;
44
+ Ok(())
45
+ }
46
+ }
47
+
48
+ /// Wait until all engines subscribe to the coordinator broadcast socket.
49
+ async fn wait_for_engine_subscriptions(
50
+ input_socket: &mut XPubSocket,
51
+ engine_count: usize,
52
+ ready_timeout: Duration,
53
+ ) -> Result<()> {
54
+ let mut received = 0;
55
+ while received < engine_count {
56
+ let message =
57
+ tokio::time::timeout(ready_timeout, input_socket.recv()).await.map_err(|_| {
58
+ Error::HandshakeTimeout {
59
+ stage: "coordinator engine subscriptions",
60
+ timeout: ready_timeout,
61
+ }
62
+ })??;
63
+ if message.len() != 1 {
64
+ bail_unexpected_handshake_message!(
65
+ "expected 1 frame for coordinator subscription, got {}",
66
+ message.len()
67
+ );
68
+ }
69
+
70
+ let frame = message
71
+ .into_vec()
72
+ .into_iter()
73
+ .next()
74
+ .expect("single-frame coordinator subscription message");
75
+ if frame.as_ref() != [0x01] {
76
+ bail_unexpected_handshake_message!(
77
+ "expected coordinator subscription frame [0x01], got {:?}",
78
+ frame.as_ref()
79
+ );
80
+ }
81
+ received += 1;
82
+ }
83
+
84
+ Ok(())
85
+ }
86
+
87
+ /// Send the coordinator READY marker to all subscribed engines.
88
+ async fn send_ready_to_engines(input_socket: &mut XPubSocket) -> Result<()> {
89
+ input_socket.send(ZmqMessage::from(Bytes::from_static(b"READY"))).await?;
90
+ Ok(())
91
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/coordinator/external.rs ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::sync::Arc;
2
+
3
+ use serde_tuple::{Deserialize_tuple, Serialize_tuple};
4
+ use thiserror_ext::AsReport;
5
+ use tokio::sync::mpsc;
6
+ use tracing::{debug, warn};
7
+ use zeromq::prelude::{SocketRecv, SocketSend};
8
+ use zeromq::{XSubSocket, ZmqMessage};
9
+
10
+ use crate::client::imp::ClientInner;
11
+ use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
12
+ use crate::error::{Error, Result, bail_unexpected_coordinator_output};
13
+ use crate::protocol::{OpaqueValue, decode_msgpack, encode_msgpack};
14
+
15
+ /// Frontend-to-coordinator wakeup message sent when the first request arrives
16
+ /// while all engines are paused.
17
+ ///
18
+ /// This matches the frontend-side msgpack tuple sent by Python
19
+ /// `DPAsyncMPClient._ensure_stats_update_task` to the coordinator front socket.
20
+ ///
21
+ /// Original Python definition:
22
+ /// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/core_client.py#L1230-L1236>
23
+ #[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
24
+ struct CoordinatorWakeupMessage {
25
+ /// Engine index that already has the triggering request and should be
26
+ /// excluded from the coordinator's `START_DP_WAVE` rebroadcast.
27
+ exclude_engine_index: u32,
28
+ /// DP wave number observed by the frontend when the request was admitted.
29
+ wave: u32,
30
+ }
31
+
32
+ /// Coordinator-to-frontend state publish received on the front-side coordinator
33
+ /// socket.
34
+ ///
35
+ /// This matches the msgpack tuple periodically published by Python
36
+ /// `DPCoordinatorProc.run_coordinator` to all connected frontends.
37
+ ///
38
+ /// Original Python definitions:
39
+ /// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/coordinator.py#L282-L283>
40
+ /// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/coordinator.py#L445-L447>
41
+ #[derive(Debug, Clone, PartialEq, Deserialize_tuple)]
42
+ struct CoordinatorStateUpdate {
43
+ /// Global per-engine request counts published by the coordinator.
44
+ ///
45
+ /// The Rust bootstrapped external-coordinator path preserves this field for
46
+ /// wire compatibility but intentionally ignores it for routing decisions.
47
+ counts: OpaqueValue,
48
+ /// Current global DP wave number stamped onto newly admitted requests.
49
+ wave: u32,
50
+ /// Whether engines are currently running (`true`) or paused (`false`).
51
+ engines_running: bool,
52
+ }
53
+
54
+ /// Background half of an external Python-owned coordinator connection.
55
+ ///
56
+ /// This owns the command receiver and one frontend-facing XSUB socket. It
57
+ /// mirrors the subset of Python's coordinator protocol needed by the Rust
58
+ /// bootstrapped frontend: receive `(counts, wave, running)` publishes, ignore
59
+ /// `counts`, and send `(exclude_engine_index, wave)` wakeup messages when the
60
+ /// first request arrives while engines are paused.
61
+ pub(crate) struct ExternalCoordinatorService {
62
+ state: Arc<CoordinatorState>,
63
+ command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
64
+ socket: XSubSocket,
65
+ }
66
+
67
+ impl ExternalCoordinatorService {
68
+ pub(super) fn new(
69
+ state: Arc<CoordinatorState>,
70
+ command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
71
+ socket: XSubSocket,
72
+ ) -> Self {
73
+ Self {
74
+ state,
75
+ command_rx,
76
+ socket,
77
+ }
78
+ }
79
+
80
+ /// Apply one frontend-originated command to the external coordinator state
81
+ /// machine.
82
+ async fn handle_command(&mut self, command: CoordinatorCommand) -> Result<()> {
83
+ match command {
84
+ CoordinatorCommand::FirstRequest {
85
+ target_engine_id,
86
+ wave,
87
+ } => {
88
+ let target_engine_index = target_engine_id.engine_index().ok_or_else(|| {
89
+ Error::UnsupportedCoordinatorEngineId {
90
+ engine_id: target_engine_id.to_vec(),
91
+ }
92
+ })?;
93
+ debug!(
94
+ wave,
95
+ exclude_engine_index = target_engine_index,
96
+ "notifying external coordinator about first request while engines were paused"
97
+ );
98
+ let payload = encode_msgpack(&CoordinatorWakeupMessage {
99
+ exclude_engine_index: target_engine_index,
100
+ wave,
101
+ })?;
102
+ self.socket.send(ZmqMessage::from(payload)).await?;
103
+ }
104
+ }
105
+ Ok(())
106
+ }
107
+
108
+ /// Apply one publish received from the xsub socket containing a coordinator
109
+ /// state update.
110
+ async fn handle_publish(&mut self, message: ZmqMessage) -> Result<()> {
111
+ let frames = message.into_vec();
112
+ if frames.len() != 1 {
113
+ bail_unexpected_coordinator_output!(
114
+ "received malformed external coordinator publish with {} frame(s)",
115
+ frames.len()
116
+ );
117
+ }
118
+
119
+ let update: CoordinatorStateUpdate = decode_msgpack(&frames[0])?;
120
+
121
+ let mut state = self.state.lock();
122
+ let previous_wave = state.current_wave;
123
+ let previous_engines_running = state.engines_running;
124
+ state.current_wave = update.wave;
125
+ state.engines_running = update.engines_running;
126
+ debug!(
127
+ previous_wave,
128
+ wave = update.wave,
129
+ previous_engines_running,
130
+ engines_running = update.engines_running,
131
+ "applied external coordinator state update"
132
+ );
133
+ Ok(())
134
+ }
135
+
136
+ /// Drive the coordinator event loop until either side of the control plane
137
+ /// is closed or a fatal error is observed.
138
+ pub(crate) async fn run(mut self, inner: Arc<ClientInner>) {
139
+ let result: Result<()> = async {
140
+ loop {
141
+ tokio::select! {
142
+ // Received frontend-originated command from the handle.
143
+ command = self.command_rx.recv() => {
144
+ let Some(command) = command else {
145
+ warn!("external coordinator command channel closed, shutting down service");
146
+ return Ok(());
147
+ };
148
+ self.handle_command(command).await?;
149
+ }
150
+ // Received publish from the external coordinator socket.
151
+ publish = self.socket.recv() => {
152
+ let publish = publish.map_err(Error::from)?;
153
+ self.handle_publish(publish).await?;
154
+ }
155
+ }
156
+ }
157
+ }
158
+ .await;
159
+ let Err(error) = result else { return };
160
+
161
+ warn!(
162
+ error = %error.as_report(),
163
+ "external coordinator service exiting with error"
164
+ );
165
+ inner.close_registries(Arc::new(error));
166
+ }
167
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/coordinator/handle.rs ADDED
@@ -0,0 +1,144 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::sync::Arc;
2
+
3
+ use parking_lot::Mutex;
4
+ use tokio::sync::mpsc;
5
+ use zeromq::prelude::Socket;
6
+ use zeromq::{XPubSocket, XSubSocket};
7
+
8
+ use crate::coordinator::external::ExternalCoordinatorService;
9
+ use crate::coordinator::inproc::InProcCoordinatorRunner;
10
+ use crate::error::{Error, Result, bail_control_closed};
11
+ use crate::transport::EngineId;
12
+
13
+ /// Snapshot to the coordinator state for request routing and stamping.
14
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
15
+ pub(crate) struct CoordinatorStateSnapshot {
16
+ /// The current DP wave, which will be stamped on outgoing requests.
17
+ pub current_wave: u32,
18
+ /// Whether the engines are currently running or paused, which determines if
19
+ /// the frontend must trigger a new wave on the next request.
20
+ pub engines_running: bool,
21
+ }
22
+
23
+ impl CoordinatorStateSnapshot {
24
+ /// Resume the engines for a `FirstRequest` and return the wave to broadcast
25
+ /// and the engine to exclude from the wakeup.
26
+ ///
27
+ /// The request may have been stamped with a `request_wave` older than
28
+ /// `current_wave` if a `WaveComplete` advanced it after the command was
29
+ /// enqueued. Such a request still needs serving, so the current wave is
30
+ /// broadcast to every engine (`exclude = None`); the wave is never rewound.
31
+ /// A non-stale request excludes the engine that already received it. Mirrors
32
+ /// the Python coordinator's front-end path.
33
+ pub(crate) fn start_wave_for_first_request(
34
+ &mut self,
35
+ request_wave: u32,
36
+ target_engine_index: u32,
37
+ ) -> (u32, Option<u32>) {
38
+ self.engines_running = true;
39
+ let exclude = (request_wave >= self.current_wave).then_some(target_engine_index);
40
+ (self.current_wave, exclude)
41
+ }
42
+ }
43
+
44
+ /// Shared in-process coordinator state.
45
+ pub(crate) type CoordinatorState = Mutex<CoordinatorStateSnapshot>;
46
+
47
+ /// Commands sent from the frontend request path into the background runner.
48
+ #[derive(Debug)]
49
+ pub(crate) enum CoordinatorCommand {
50
+ /// The first request arrived while all engines were paused.
51
+ ///
52
+ /// The coordinator should broadcast `START_DP_WAVE` with the current wave
53
+ /// and the target engine index as the excluded engine.
54
+ FirstRequest {
55
+ target_engine_id: EngineId,
56
+ wave: u32,
57
+ },
58
+ }
59
+
60
+ /// Frontend-facing coordinator handle used by `EngineCoreClient::call()`.
61
+ ///
62
+ /// This side stays intentionally small: it can read the latest wave snapshot
63
+ /// and enqueue a `FirstRequest` transition when the request path observes the
64
+ /// system in the paused state.
65
+ #[derive(Clone)]
66
+ pub(crate) struct CoordinatorHandle {
67
+ state: Arc<CoordinatorState>,
68
+ command_tx: mpsc::UnboundedSender<CoordinatorCommand>,
69
+ }
70
+
71
+ impl CoordinatorHandle {
72
+ fn new_parts() -> (
73
+ Self,
74
+ Arc<CoordinatorState>,
75
+ mpsc::UnboundedReceiver<CoordinatorCommand>,
76
+ ) {
77
+ let state = Arc::new(Mutex::new(CoordinatorStateSnapshot {
78
+ current_wave: 0,
79
+ engines_running: false,
80
+ }));
81
+ let (command_tx, command_rx) = mpsc::unbounded_channel();
82
+ (
83
+ Self {
84
+ state: state.clone(),
85
+ command_tx,
86
+ },
87
+ state,
88
+ command_rx,
89
+ )
90
+ }
91
+
92
+ /// Build the paired frontend handle and background runner around one
93
+ /// engine-facing coordinator broadcast socket.
94
+ pub(crate) fn new_inproc(coordinator_input: XPubSocket) -> (Self, InProcCoordinatorRunner) {
95
+ let (handle, state, command_rx) = Self::new_parts();
96
+ (
97
+ handle,
98
+ InProcCoordinatorRunner::new(state, command_rx, coordinator_input),
99
+ )
100
+ }
101
+
102
+ /// Build the paired frontend handle and background service around an
103
+ /// external Python-owned frontend-side coordinator socket.
104
+ pub(crate) async fn connect_external(
105
+ coordinator_address: &str,
106
+ ) -> Result<(Self, ExternalCoordinatorService)> {
107
+ let (handle, state, command_rx) = Self::new_parts();
108
+ let mut socket = XSubSocket::new();
109
+ socket.connect(coordinator_address).await?;
110
+ socket.subscribe("").await?;
111
+ Ok((
112
+ handle,
113
+ ExternalCoordinatorService::new(state, command_rx, socket),
114
+ ))
115
+ }
116
+
117
+ /// Snapshot the coordinator state for request routing and stamping.
118
+ pub(crate) fn snapshot(&self) -> CoordinatorStateSnapshot {
119
+ *self.state.lock()
120
+ }
121
+
122
+ /// Notify the runner that a new request arrived while engines were paused.
123
+ ///
124
+ /// The handle flips `engines_running` optimistically so concurrent request
125
+ /// submissions coalesce behind one `START_DP_WAVE` broadcast instead of all
126
+ /// trying to trigger the wave independently.
127
+ pub(crate) fn notify_first_request(&self, target_engine_id: EngineId) -> Result<()> {
128
+ let mut state = self.state.lock();
129
+ if state.engines_running {
130
+ return Ok(());
131
+ }
132
+
133
+ let command = CoordinatorCommand::FirstRequest {
134
+ target_engine_id,
135
+ wave: state.current_wave,
136
+ };
137
+ if self.command_tx.send(command).is_err() {
138
+ bail_control_closed!("in-process coordinator command channel already shut down");
139
+ }
140
+
141
+ state.engines_running = true;
142
+ Ok(())
143
+ }
144
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/coordinator/inproc.rs ADDED
@@ -0,0 +1,257 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::sync::Arc;
2
+
3
+ use serde_tuple::{Deserialize_tuple, Serialize_tuple};
4
+ use thiserror_ext::AsReport;
5
+ use tokio::sync::mpsc;
6
+ use tracing::{debug, warn};
7
+ use zeromq::prelude::SocketSend;
8
+ use zeromq::{XPubSocket, ZmqMessage};
9
+
10
+ use crate::client::imp::ClientInner;
11
+ use crate::coordinator::handle::{CoordinatorCommand, CoordinatorState};
12
+ use crate::error::{Error, Result, bail_unexpected_coordinator_output};
13
+ use crate::protocol::encode_msgpack;
14
+ use crate::protocol::output::{DpControlMessage, DpControlOutput, EngineCoreOutputs};
15
+ use crate::protocol::request::EngineCoreRequestType;
16
+
17
+ /// Coordinator-to-engine `START_DP_WAVE` control payload encoded on the
18
+ /// engine-facing coordinator socket.
19
+ ///
20
+ /// This matches the msgpack tuple broadcast by Python
21
+ /// `DPCoordinatorProc._send_start_wave`.
22
+ ///
23
+ /// Original Python definition:
24
+ /// <https://github.com/vllm-project/vllm/blob/694449050f8dac3d9853e97e518b4a43ec52106a/vllm/v1/engine/coordinator.py#L453-L459>
25
+ #[derive(Debug, Clone, PartialEq, Eq, Serialize_tuple, Deserialize_tuple)]
26
+ struct StartDpWaveMessage {
27
+ /// DP wave number that all engines should start processing.
28
+ wave: u32,
29
+ /// Engine index that already received the triggering request and so does not
30
+ /// need an extra wakeup. `None` wakes every engine (used when the triggering
31
+ /// request was for a stale wave).
32
+ exclude_engine_index: Option<u32>,
33
+ }
34
+
35
+ /// Background half of the in-process coordinator.
36
+ ///
37
+ /// This owns the command receiver and the engine-facing coordinator input
38
+ /// socket. It is the single place where wave transitions are serialized and
39
+ /// where `START_DP_WAVE` broadcasts are emitted.
40
+ pub(crate) struct InProcCoordinatorRunner {
41
+ state: Arc<CoordinatorState>,
42
+ command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
43
+ coordinator_input: XPubSocket,
44
+ }
45
+
46
+ impl InProcCoordinatorRunner {
47
+ pub(super) fn new(
48
+ state: Arc<CoordinatorState>,
49
+ command_rx: mpsc::UnboundedReceiver<CoordinatorCommand>,
50
+ coordinator_input: XPubSocket,
51
+ ) -> Self {
52
+ Self {
53
+ state,
54
+ command_rx,
55
+ coordinator_input,
56
+ }
57
+ }
58
+
59
+ /// Broadcast Python-compatible `START_DP_WAVE` to all connected engines.
60
+ async fn broadcast_start_wave(
61
+ &mut self,
62
+ wave: u32,
63
+ exclude_engine_index: Option<u32>,
64
+ ) -> Result<()> {
65
+ let payload = encode_msgpack(&StartDpWaveMessage {
66
+ wave,
67
+ exclude_engine_index,
68
+ })?;
69
+ self.coordinator_input
70
+ .send(
71
+ ZmqMessage::try_from(vec![
72
+ EngineCoreRequestType::StartDpWave.to_frame(),
73
+ payload.into(),
74
+ ])
75
+ .expect("coordinator START_DP_WAVE message must contain two frames"),
76
+ )
77
+ .await?;
78
+ Ok(())
79
+ }
80
+
81
+ /// Apply one frontend-originated command to the coordinator state machine.
82
+ async fn handle_command(&mut self, command: CoordinatorCommand) -> Result<()> {
83
+ match command {
84
+ CoordinatorCommand::FirstRequest {
85
+ target_engine_id,
86
+ wave,
87
+ } => {
88
+ let target_engine_index = target_engine_id.engine_index().ok_or_else(|| {
89
+ Error::UnsupportedCoordinatorEngineId {
90
+ engine_id: target_engine_id.to_vec(),
91
+ }
92
+ })?;
93
+ let (current_wave, exclude) = {
94
+ let mut state = self.state.lock();
95
+ state.start_wave_for_first_request(wave, target_engine_index)
96
+ };
97
+ debug!(
98
+ current_wave,
99
+ request_wave = wave,
100
+ ?exclude,
101
+ "starting DP wave after first request while engines were paused"
102
+ );
103
+ self.broadcast_start_wave(current_wave, exclude).await?;
104
+ }
105
+ }
106
+ Ok(())
107
+ }
108
+
109
+ /// Apply one engine-originated control output to the coordinator state
110
+ /// machine.
111
+ async fn handle_outputs(&mut self, outputs: EngineCoreOutputs) -> Result<()> {
112
+ match outputs {
113
+ EngineCoreOutputs::RequestBatch(batch)
114
+ if batch.outputs.is_empty() && batch.finished_requests.is_none() =>
115
+ {
116
+ // Stats-only output for coordinator.
117
+ // Ignore since the Rust coordinator doesn't track stats for
118
+ // routing decisions.
119
+ }
120
+ EngineCoreOutputs::DpControl(DpControlOutput {
121
+ engine_index,
122
+ control,
123
+ ..
124
+ }) => match control {
125
+ // The engines signals they completed the current wave and are now paused.
126
+ // Advance the current wave and mark the state as paused.
127
+ DpControlMessage::WaveComplete(wave) => {
128
+ let mut state = self.state.lock();
129
+ if wave >= state.current_wave {
130
+ let next_wave = wave + 1;
131
+ debug!(
132
+ wave,
133
+ next_wave,
134
+ "DP wave finished; pausing engines and advancing coordinator state"
135
+ );
136
+ state.current_wave = wave + 1;
137
+ state.engines_running = false;
138
+ }
139
+ }
140
+ // An engine requests to start the wave.
141
+ // Rebroadcast the wave to all engines except for the originated one.
142
+ DpControlMessage::StartWave(wave) => {
143
+ let should_broadcast = {
144
+ let mut state = self.state.lock();
145
+ if wave > state.current_wave
146
+ || (wave == state.current_wave && !state.engines_running)
147
+ {
148
+ state.current_wave = wave;
149
+ state.engines_running = true;
150
+ true
151
+ } else {
152
+ false
153
+ }
154
+ };
155
+ if should_broadcast {
156
+ debug!(
157
+ wave,
158
+ exclude_engine_index = engine_index,
159
+ "starting DP wave after stale-wave notification from engine"
160
+ );
161
+ self.broadcast_start_wave(wave, Some(engine_index)).await?;
162
+ }
163
+ }
164
+ },
165
+ other => {
166
+ bail_unexpected_coordinator_output!(
167
+ "received non-control output on coordinator path: {other:?}"
168
+ );
169
+ }
170
+ }
171
+ Ok(())
172
+ }
173
+
174
+ /// Drive the coordinator event loop until either side of the control plane
175
+ /// is closed or a fatal error is observed.
176
+ ///
177
+ /// Any fatal error closes the main client registries so request streams and
178
+ /// future calls observe a stable shutdown cause.
179
+ pub(crate) async fn run(
180
+ mut self,
181
+ mut output_rx: mpsc::Receiver<Result<EngineCoreOutputs>>,
182
+ inner: Arc<ClientInner>,
183
+ ) {
184
+ let result: Result<()> = async {
185
+ loop {
186
+ tokio::select! {
187
+ // Received frontend-originated command from the handle.
188
+ command = self.command_rx.recv() => {
189
+ let Some(command) = command else {
190
+ warn!("coordinator command channel closed, shutting down coordinator runner");
191
+ return Ok(());
192
+ };
193
+ self.handle_command(command).await?;
194
+ }
195
+ // Received engine-originated control output from the coordinator socket.
196
+ outputs = output_rx.recv() => {
197
+ let Some(outputs) = outputs else {
198
+ warn!("coordinator output channel closed, shutting down coordinator runner");
199
+ return Ok(());
200
+ };
201
+ self.handle_outputs(outputs?).await?;
202
+ }
203
+ }
204
+ }
205
+ }
206
+ .await;
207
+ let Err(error) = result else { return };
208
+
209
+ warn!(error = %error.as_report(), "coordinator runner exiting with error");
210
+ inner.close_registries(Arc::new(error));
211
+ }
212
+ }
213
+
214
+ #[cfg(test)]
215
+ mod tests {
216
+ use crate::coordinator::handle::CoordinatorStateSnapshot;
217
+
218
+ /// A `FirstRequest` for the current wave starts that wave and excludes the
219
+ /// engine that already received the triggering request.
220
+ #[test]
221
+ fn first_request_for_current_wave_excludes_target() {
222
+ let mut state = CoordinatorStateSnapshot {
223
+ current_wave: 3,
224
+ engines_running: false,
225
+ };
226
+
227
+ let (wave, exclude) = state.start_wave_for_first_request(3, 2);
228
+
229
+ assert_eq!(wave, 3);
230
+ assert_eq!(exclude, Some(2));
231
+ assert!(state.engines_running);
232
+ assert_eq!(state.current_wave, 3);
233
+ }
234
+
235
+ /// A `FirstRequest` whose wave was superseded by a racing `WaveComplete`
236
+ /// (`request_wave < current_wave`) must still start the request's wave: it
237
+ /// broadcasts the current wave and wakes every engine (`exclude = None`)
238
+ /// rather than rewinding the wave or dropping the request.
239
+ #[test]
240
+ fn stale_first_request_starts_current_wave_for_all_engines() {
241
+ let mut state = CoordinatorStateSnapshot {
242
+ current_wave: 4,
243
+ engines_running: false,
244
+ };
245
+
246
+ // Request stamped with wave 3 while the coordinator already advanced to 4.
247
+ let (wave, exclude) = state.start_wave_for_first_request(3, 2);
248
+
249
+ assert_eq!(
250
+ wave, 4,
251
+ "must broadcast the current wave, not the stale one"
252
+ );
253
+ assert_eq!(exclude, None, "a stale request must wake every engine");
254
+ assert!(state.engines_running);
255
+ assert_eq!(state.current_wave, 4, "wave must not be rewound");
256
+ }
257
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/coordinator/mod.rs ADDED
@@ -0,0 +1,7 @@
 
 
 
 
 
 
 
 
1
+ mod bootstrap;
2
+ mod external;
3
+ mod handle;
4
+ mod inproc;
5
+
6
+ pub(crate) use bootstrap::CoordinatorBootstrap;
7
+ pub(crate) use handle::CoordinatorHandle;
asian_ai/performance/vLLM/rust/src/engine-core-client/src/error.rs ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::sync::Arc;
2
+ use std::time::Duration;
3
+
4
+ use thiserror::Error;
5
+ use thiserror_ext::Macro;
6
+
7
+ use crate::protocol::utility::UtilityCallId;
8
+
9
+ pub type Result<T> = std::result::Result<T, Error>;
10
+
11
+ /// Public error type for the Rust engine-core client.
12
+ #[derive(Debug, Error, Macro)]
13
+ pub enum Error {
14
+ #[error("messagepack encode failed for {target_type}: {message}")]
15
+ Encode {
16
+ target_type: &'static str,
17
+ message: String,
18
+ },
19
+ #[error("messagepack decode failed for {target_type}: {message}")]
20
+ Decode {
21
+ target_type: &'static str,
22
+ message: String,
23
+ },
24
+ #[error("messagepack value decode failed")]
25
+ ValueDecode(#[from] rmpv::decode::Error),
26
+ #[error("messagepack ext value decode failed: {message}")]
27
+ ExtValueDecode { message: String },
28
+ #[error("invalid structured outputs params: {message}")]
29
+ InvalidStructuredOutputsParams { message: String },
30
+ #[error("io error")]
31
+ Io(#[from] std::io::Error),
32
+ #[error("transport error")]
33
+ Transport(#[from] zeromq::ZmqError),
34
+ #[error("ZMQ runtime task failed")]
35
+ ZmqRuntimeTask(#[from] tokio::task::JoinError),
36
+ #[error("engine core reported fatal failure")]
37
+ EngineCoreDead,
38
+ #[error("startup handshake timed out while waiting for {stage} after {timeout:?}")]
39
+ HandshakeTimeout {
40
+ stage: &'static str,
41
+ timeout: Duration,
42
+ },
43
+ #[error("engine input registration timed out after {timeout:?}")]
44
+ InputRegistrationTimeout { timeout: Duration },
45
+ #[error("unexpected engine id in startup handshake: expected {expected:?}, got {actual:?}")]
46
+ UnexpectedHandshakeIdentity { expected: Vec<u8>, actual: Vec<u8> },
47
+ #[error("unexpected startup handshake message: {message}")]
48
+ UnexpectedHandshakeMessage { message: String },
49
+ #[error("unexpected non-control output on coordinator path: {message}")]
50
+ UnexpectedCoordinatorOutput { message: String },
51
+ #[error("unexpected output on main dispatcher path: {message}")]
52
+ UnexpectedDispatcherOutput { message: String },
53
+ #[error("coordinator requires a Python-compatible two-byte engine id, got {engine_id:?}")]
54
+ UnsupportedCoordinatorEngineId { engine_id: Vec<u8> },
55
+ #[error("unsupported auxiliary frame(s): expected 1 frame, got {frame_count}")]
56
+ UnsupportedAuxFrames { frame_count: usize },
57
+ #[error("external coordinator mode is not implemented yet")]
58
+ UnsupportedExternalCoordinator,
59
+ #[error("unsupported field `{field}` in {context}")]
60
+ UnsupportedField {
61
+ context: &'static str,
62
+ field: &'static str,
63
+ },
64
+ #[error("engine control channel closed unexpectedly: {message}")]
65
+ ControlClosed { message: String },
66
+ #[error("request `{request_id}` is already in flight")]
67
+ DuplicateRequestId { request_id: String },
68
+ #[error("data parallel rank {rank} is out of range for {num_engines} engine(s)")]
69
+ InvalidDataParallelRank { rank: u32, num_engines: u32 },
70
+ #[error("engine-core output dispatcher closed: {message}")]
71
+ DispatcherClosed { message: String },
72
+ #[error("engine-core client is closed: {message}")]
73
+ ClientClosed { message: String },
74
+ #[error("request output stream for `{request_id}` closed unexpectedly")]
75
+ RequestStreamClosed { request_id: String },
76
+ #[error("utility call `{method}` failed (call_id={call_id}): {message}")]
77
+ UtilityCallFailed {
78
+ method: String,
79
+ call_id: UtilityCallId,
80
+ message: String,
81
+ },
82
+ #[error("utility call `{method}` returned an invalid result (call_id={call_id}): {message}")]
83
+ UtilityResultDecode {
84
+ method: String,
85
+ call_id: UtilityCallId,
86
+ message: String,
87
+ },
88
+ #[error("utility call `{method}` closed unexpectedly (call_id={call_id})")]
89
+ UtilityCallClosed { method: String, call_id: u64 },
90
+ #[error("utility call `{method}` returned inconsistent results across engines: {values}")]
91
+ InconsistentUtilityResults { method: String, values: String },
92
+
93
+ /// A special variant to allow cloning the same error.
94
+ #[error(transparent)]
95
+ Shared(Arc<Self>),
96
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/lib.rs ADDED
@@ -0,0 +1,20 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ mod client;
2
+ mod coordinator;
3
+ mod error;
4
+ mod metrics;
5
+ pub mod mock_engine;
6
+ pub mod protocol;
7
+ pub mod runtime;
8
+ #[cfg(any(test, feature = "test-util"))]
9
+ pub mod test_utils;
10
+ mod transport;
11
+
12
+ pub use client::{
13
+ AbortCause, CoordinatorMode, EngineCoreClient, EngineCoreClientConfig, EngineCoreOutputStream,
14
+ EngineCoreStreamOutput, TransportMode,
15
+ };
16
+ pub use error::{Error, Result};
17
+ pub use transport::{ENGINE_CORE_DEAD_SENTINEL, EngineId};
18
+
19
+ #[cfg(test)]
20
+ mod tests;
asian_ai/performance/vLLM/rust/src/engine-core-client/src/metrics.rs ADDED
@@ -0,0 +1,259 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::collections::BTreeSet;
2
+ use std::time::{SystemTime, UNIX_EPOCH};
3
+
4
+ use vllm_metrics::{
5
+ EngineLabels, EnginePositionLabels, LoraAdapterNames, LoraInfoLabels, SchedulerMetrics,
6
+ WaitingReasonLabels,
7
+ };
8
+
9
+ use crate::protocol::stats::SchedulerStats;
10
+
11
+ const WAITING_REASON_CAPACITY: &str = "capacity";
12
+ const WAITING_REASON_DEFERRED: &str = "deferred";
13
+
14
+ /// Record the scheduler-stats-backed metrics for one engine at one point in
15
+ /// time.
16
+ pub(crate) fn record_scheduler_stats(
17
+ metrics: &SchedulerMetrics,
18
+ model_name: impl Into<String>,
19
+ engine: u32,
20
+ stats: &SchedulerStats,
21
+ ) {
22
+ let model_name = model_name.into();
23
+ let labels = EngineLabels {
24
+ model_name: model_name.clone(),
25
+ engine,
26
+ };
27
+
28
+ // Scheduler state gauges.
29
+ metrics.scheduler_running.get_or_create(&labels).set(stats.num_running_reqs);
30
+ metrics
31
+ .scheduler_waiting
32
+ .get_or_create(&labels)
33
+ .set(stats.num_waiting_reqs + stats.num_skipped_waiting_reqs);
34
+ metrics
35
+ .scheduler_waiting_by_reason
36
+ .get_or_create(&WaitingReasonLabels {
37
+ model_name: model_name.clone(),
38
+ engine,
39
+ reason: WAITING_REASON_CAPACITY,
40
+ })
41
+ .set(stats.num_waiting_reqs);
42
+ metrics
43
+ .scheduler_waiting_by_reason
44
+ .get_or_create(&WaitingReasonLabels {
45
+ model_name: model_name.clone(),
46
+ engine,
47
+ reason: WAITING_REASON_DEFERRED,
48
+ })
49
+ .set(stats.num_skipped_waiting_reqs);
50
+ metrics.kv_cache_usage.get_or_create(&labels).set(stats.kv_cache_usage);
51
+
52
+ // Prefix-cache counters, including the connector-backed external cache path.
53
+ metrics
54
+ .prefix_cache_queries
55
+ .get_or_create(&labels)
56
+ .inc_by(stats.prefix_cache_stats.base.queries);
57
+ metrics
58
+ .prefix_cache_hits
59
+ .get_or_create(&labels)
60
+ .inc_by(stats.prefix_cache_stats.base.hits);
61
+
62
+ if let Some(connector_prefix_cache_stats) = &stats.connector_prefix_cache_stats {
63
+ metrics
64
+ .external_prefix_cache_queries
65
+ .get_or_create(&labels)
66
+ .inc_by(connector_prefix_cache_stats.base.queries);
67
+ metrics
68
+ .external_prefix_cache_hits
69
+ .get_or_create(&labels)
70
+ .inc_by(connector_prefix_cache_stats.base.hits);
71
+ }
72
+
73
+ // Speculative decoding counters.
74
+ if let Some(spec_decoding_stats) = &stats.spec_decoding_stats {
75
+ metrics
76
+ .spec_decode_num_drafts
77
+ .get_or_create(&labels)
78
+ .inc_by(spec_decoding_stats.num_drafts);
79
+ metrics
80
+ .spec_decode_num_draft_tokens
81
+ .get_or_create(&labels)
82
+ .inc_by(spec_decoding_stats.num_draft_tokens);
83
+ metrics
84
+ .spec_decode_num_accepted_tokens
85
+ .get_or_create(&labels)
86
+ .inc_by(spec_decoding_stats.num_accepted_tokens);
87
+ metrics.log_stats.get_or_create(&labels).observe_spec_decode(
88
+ spec_decoding_stats.num_drafts,
89
+ &spec_decoding_stats.num_accepted_tokens_per_pos,
90
+ );
91
+
92
+ for (position, accepted_tokens) in
93
+ spec_decoding_stats.num_accepted_tokens_per_pos.iter().copied().enumerate()
94
+ {
95
+ metrics
96
+ .spec_decode_num_accepted_tokens_per_pos
97
+ .get_or_create(&EnginePositionLabels {
98
+ model_name: model_name.clone(),
99
+ engine,
100
+ position: position as u32,
101
+ })
102
+ .inc_by(accepted_tokens);
103
+ }
104
+ }
105
+
106
+ // Per-engine performance / MFU counters.
107
+ if let Some(perf_stats) = &stats.perf_stats
108
+ && (perf_stats.num_flops_per_gpu != 0
109
+ || perf_stats.num_read_bytes_per_gpu != 0
110
+ || perf_stats.num_write_bytes_per_gpu != 0)
111
+ {
112
+ metrics
113
+ .estimated_flops_per_gpu
114
+ .get_or_create(&labels)
115
+ .inc_by(perf_stats.num_flops_per_gpu);
116
+ metrics
117
+ .estimated_read_bytes_per_gpu
118
+ .get_or_create(&labels)
119
+ .inc_by(perf_stats.num_read_bytes_per_gpu);
120
+ metrics
121
+ .estimated_write_bytes_per_gpu
122
+ .get_or_create(&labels)
123
+ .inc_by(perf_stats.num_write_bytes_per_gpu);
124
+ }
125
+
126
+ if let Some(cudagraph_stats) = &stats.cudagraph_stats {
127
+ metrics.log_stats.get_or_create(&labels).observe_cudagraph(
128
+ cudagraph_stats.num_unpadded_tokens,
129
+ cudagraph_stats.num_padded_tokens,
130
+ cudagraph_stats.num_paddings,
131
+ &cudagraph_stats.runtime_mode,
132
+ );
133
+ }
134
+
135
+ // Sampled KV-cache residency histograms.
136
+ if !stats.kv_cache_eviction_events.is_empty() {
137
+ let kv_block_lifetime_seconds = metrics.kv_block_lifetime_seconds.get_or_create(&labels);
138
+ let kv_block_idle_before_evict_seconds =
139
+ metrics.kv_block_idle_before_evict_seconds.get_or_create(&labels);
140
+ let kv_block_reuse_gap_seconds = metrics.kv_block_reuse_gap_seconds.get_or_create(&labels);
141
+
142
+ for event in &stats.kv_cache_eviction_events {
143
+ kv_block_lifetime_seconds.observe(event.lifetime_seconds);
144
+ kv_block_idle_before_evict_seconds.observe(event.idle_seconds);
145
+ for reuse_gap_seconds in &event.reuse_gaps_seconds {
146
+ kv_block_reuse_gap_seconds.observe(*reuse_gap_seconds);
147
+ }
148
+ }
149
+ }
150
+ }
151
+
152
+ /// Exports `vllm:lora_requests_info` as a single series covering all LoRA
153
+ /// requests tracked by this client across every engine in the replica.
154
+ ///
155
+ /// The engine's `SchedulerStats` never carries adapter names: the Python
156
+ /// frontend fills them in from per-request lifecycle events tracked by
157
+ /// `LoRARequestStates` in `vllm/v1/engine/output_processor.py`. The Rust
158
+ /// frontend mirrors that, deriving the sets from the request registry.
159
+ #[derive(Default)]
160
+ pub(crate) struct LoraInfoExporter {
161
+ current: Option<LoraInfoLabels>,
162
+ }
163
+
164
+ impl LoraInfoExporter {
165
+ pub(crate) fn update(
166
+ &mut self,
167
+ metrics: &SchedulerMetrics,
168
+ running: BTreeSet<String>,
169
+ waiting: BTreeSet<String>,
170
+ ) {
171
+ let next = (!running.is_empty() || !waiting.is_empty()).then_some(LoraInfoLabels {
172
+ running_lora_adapters: LoraAdapterNames(running),
173
+ waiting_lora_adapters: LoraAdapterNames(waiting),
174
+ });
175
+
176
+ if self.current != next
177
+ && let Some(prev) = &self.current
178
+ {
179
+ metrics.lora_info.remove(prev);
180
+ }
181
+
182
+ // Python sets this gauge to the current time on every record.
183
+ if let Some(labels) = &next {
184
+ metrics.lora_info.get_or_create(labels).set(now_unix_secs());
185
+ }
186
+
187
+ self.current = next;
188
+ }
189
+ }
190
+
191
+ fn now_unix_secs() -> f64 {
192
+ SystemTime::now()
193
+ .duration_since(UNIX_EPOCH)
194
+ .map(|d| d.as_secs_f64())
195
+ .unwrap_or(0.0)
196
+ }
197
+
198
+ #[cfg(test)]
199
+ mod tests {
200
+ use std::collections::BTreeSet;
201
+
202
+ use expect_test::expect;
203
+ use vllm_metrics::Metrics;
204
+
205
+ use crate::metrics::LoraInfoExporter;
206
+
207
+ fn names(values: &[&str]) -> BTreeSet<String> {
208
+ values.iter().map(|name| (*name).to_string()).collect()
209
+ }
210
+
211
+ /// The `lora_requests_info` series with the non-deterministic timestamp
212
+ /// value replaced by `<ts>`, one line per series.
213
+ fn lora_series(rendered: &str) -> String {
214
+ rendered
215
+ .lines()
216
+ .filter(|l| l.starts_with("vllm:lora_requests_info{"))
217
+ .map(|l| match l.rsplit_once("} ") {
218
+ Some((labels, _value)) => format!("{labels}}} <ts>"),
219
+ None => l.to_string(),
220
+ })
221
+ .collect::<Vec<_>>()
222
+ .join("\n")
223
+ }
224
+
225
+ #[test]
226
+ fn lora_info_emits_clears_stale_and_drains() {
227
+ let metrics = Metrics::new();
228
+ let mut exporter = LoraInfoExporter::default();
229
+
230
+ // No adapters: nothing emitted.
231
+ exporter.update(&metrics.scheduler, names(&[]), names(&[]));
232
+ expect![[""]].assert_eq(&lora_series(&metrics.render().unwrap()));
233
+
234
+ // Two running (sorted), one waiting.
235
+ exporter.update(&metrics.scheduler, names(&["b", "a"]), names(&["c"]));
236
+ expect![[
237
+ r#"vllm:lora_requests_info{running_lora_adapters="a,b",waiting_lora_adapters="c"} <ts>"#
238
+ ]]
239
+ .assert_eq(&lora_series(&metrics.render().unwrap()));
240
+
241
+ // "c" gets scheduled and "d" arrives: the stale series is replaced.
242
+ exporter.update(&metrics.scheduler, names(&["a", "b", "c"]), names(&["d"]));
243
+ expect![[
244
+ r#"vllm:lora_requests_info{running_lora_adapters="a,b,c",waiting_lora_adapters="d"} <ts>"#
245
+ ]]
246
+ .assert_eq(&lora_series(&metrics.render().unwrap()));
247
+
248
+ // Everything but "d" finishes.
249
+ exporter.update(&metrics.scheduler, names(&["d"]), names(&[]));
250
+ expect![[
251
+ r#"vllm:lora_requests_info{running_lora_adapters="d",waiting_lora_adapters=""} <ts>"#
252
+ ]]
253
+ .assert_eq(&lora_series(&metrics.render().unwrap()));
254
+
255
+ // All requests done: series removed entirely.
256
+ exporter.update(&metrics.scheduler, names(&[]), names(&[]));
257
+ expect![[""]].assert_eq(&lora_series(&metrics.render().unwrap()));
258
+ }
259
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/mock_engine.rs ADDED
@@ -0,0 +1,274 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::path::Path;
2
+ use std::time::Duration;
3
+
4
+ use tokio::time::timeout;
5
+ use zeromq::prelude::{Socket, SocketRecv, SocketSend};
6
+ use zeromq::util::PeerIdentity;
7
+ use zeromq::{DealerSocket, PushSocket, SocketOptions, SubSocket, ZmqMessage};
8
+
9
+ use crate::EngineId;
10
+ use crate::error::{Error, Result, bail_unexpected_handshake_message};
11
+ use crate::protocol::dtype::ModelDtype;
12
+ use crate::protocol::handshake::{EngineCoreReadyResponse, HandshakeInitMessage, ReadyMessage};
13
+ use crate::protocol::{decode_msgpack, encode_msgpack};
14
+
15
+ /// Default model length advertised by reusable mock engine helpers.
16
+ pub const DEFAULT_MOCK_MAX_MODEL_LEN: u64 = 1024 * 1024;
17
+ /// Default KV block count advertised by reusable mock engine helpers.
18
+ pub const DEFAULT_MOCK_NUM_GPU_BLOCKS: u64 = 0;
19
+ /// Default KV block size (tokens per block)
20
+ pub const DEFAULT_MOCK_BLOCK_SIZE: u64 = 16;
21
+
22
+ /// Startup behavior for one mock engine joining a frontend.
23
+ #[derive(Debug, Clone)]
24
+ pub struct MockEngineConfig {
25
+ /// Whether the engine should advertise itself as local to the frontend.
26
+ pub local: bool,
27
+ /// Whether the engine should advertise itself as headless.
28
+ pub headless: bool,
29
+ /// Engine-ready payload reported after INIT, including max model length,
30
+ /// KV block count, and dtype.
31
+ pub ready_response: EngineCoreReadyResponse,
32
+ /// Maximum time to wait for IPC endpoints to appear before connecting.
33
+ pub connect_timeout: Duration,
34
+ }
35
+
36
+ impl Default for MockEngineConfig {
37
+ fn default() -> Self {
38
+ Self {
39
+ local: false,
40
+ headless: true,
41
+ ready_response: default_ready_response(),
42
+ connect_timeout: Duration::from_secs(5),
43
+ }
44
+ }
45
+ }
46
+
47
+ /// Construct the ready response used by the standalone mock engine CLI.
48
+ pub fn default_ready_response() -> EngineCoreReadyResponse {
49
+ EngineCoreReadyResponse {
50
+ max_model_len: DEFAULT_MOCK_MAX_MODEL_LEN,
51
+ num_gpu_blocks: DEFAULT_MOCK_NUM_GPU_BLOCKS,
52
+ block_size: DEFAULT_MOCK_BLOCK_SIZE,
53
+ dp_stats_address: None,
54
+ dtype: ModelDtype::Float32,
55
+ vllm_version: "test-vllm-version".to_string(),
56
+ world_size: 1,
57
+ data_parallel_size: 1,
58
+ kv_cache_size_tokens: None,
59
+ kv_cache_max_concurrency: None,
60
+ }
61
+ }
62
+
63
+ /// Coordinator-side sockets used by one mock engine when coordinator mode
64
+ /// is enabled.
65
+ pub struct MockCoordinatorSockets {
66
+ /// Subscription socket that receives coordinator broadcasts such as
67
+ /// `START_DP_WAVE`.
68
+ pub input_sub: SubSocket,
69
+ /// Push socket used to send coordinator-only `EngineCoreOutputs` back to
70
+ /// the frontend.
71
+ pub output_push: PushSocket,
72
+ }
73
+
74
+ /// One mock engine's connection to one frontend client.
75
+ ///
76
+ /// vLLM launches one engine-client pair per API server process. A remote
77
+ /// engine connects to every advertised input/output pair and uses the request's
78
+ /// `client_index` to route outputs back to the originating API server.
79
+ pub struct MockEngineDataSockets {
80
+ /// Socket used to receive frontend requests.
81
+ pub dealer: DealerSocket,
82
+ /// Socket used to publish normal request outputs back to the frontend.
83
+ pub push: PushSocket,
84
+ }
85
+
86
+ /// Frontend-facing sockets owned by one mock engine.
87
+ pub struct MockEngineSockets {
88
+ /// Decoded INIT message sent by the frontend during handshake.
89
+ pub init: HandshakeInitMessage,
90
+ /// Data sockets for all frontend clients in client-index order.
91
+ ///
92
+ /// For Rust frontend this will always be one socket, while for Python frontend
93
+ /// this may be multiple sockets if there are multiple API server processes.
94
+ pub data_sockets: Vec<MockEngineDataSockets>,
95
+ /// Optional coordinator sockets when the client enabled the in-process
96
+ /// coordinator.
97
+ pub coordinator: Option<MockCoordinatorSockets>,
98
+ }
99
+
100
+ /// Build a HELLO or READY handshake status payload.
101
+ fn ready_message(status: &str, config: &MockEngineConfig) -> ReadyMessage {
102
+ ReadyMessage {
103
+ status: Some(status.to_string()),
104
+ local: Some(config.local),
105
+ headless: Some(config.headless),
106
+ parallel_config_hash: None,
107
+ }
108
+ }
109
+
110
+ /// Convert an engine id into a ZMQ DEALER identity.
111
+ fn peer_identity(engine_id: impl Into<EngineId>) -> Result<PeerIdentity> {
112
+ let engine_id = engine_id.into();
113
+ PeerIdentity::try_from(engine_id.clone()).map_err(|error| Error::UnexpectedHandshakeMessage {
114
+ message: format!(
115
+ "invalid mock engine identity {:?}: {error}",
116
+ engine_id.to_vec()
117
+ ),
118
+ })
119
+ }
120
+
121
+ /// Wait for an IPC endpoint path to appear before attempting to connect.
122
+ async fn wait_for_ipc_endpoint(endpoint: &str, connect_timeout: Duration) -> Result<()> {
123
+ let Some(socket_path) = endpoint.strip_prefix("ipc://") else {
124
+ return Ok(());
125
+ };
126
+
127
+ timeout(connect_timeout, async {
128
+ while !Path::new(socket_path).exists() {
129
+ tokio::time::sleep(Duration::from_millis(20)).await;
130
+ }
131
+ })
132
+ .await
133
+ .map_err(|_| Error::HandshakeTimeout {
134
+ stage: "mock engine IPC endpoint",
135
+ timeout: connect_timeout,
136
+ })
137
+ }
138
+
139
+ /// Encode the engine-ready response sent on input socket registration.
140
+ fn ready_response_payload(config: &MockEngineConfig) -> Result<Vec<u8>> {
141
+ encode_msgpack(&config.ready_response)
142
+ }
143
+
144
+ /// Join a frontend-owned handshake endpoint and open mock engine sockets.
145
+ pub async fn connect_to_frontend(
146
+ engine_handshake: impl AsRef<str>,
147
+ engine_id: impl Into<EngineId>,
148
+ config: MockEngineConfig,
149
+ ) -> Result<MockEngineSockets> {
150
+ let engine_handshake = engine_handshake.as_ref();
151
+ wait_for_ipc_endpoint(engine_handshake, config.connect_timeout).await?;
152
+
153
+ let peer_identity = peer_identity(engine_id)?;
154
+ let mut options = SocketOptions::default();
155
+ options.peer_identity(peer_identity.clone());
156
+ let mut handshake = DealerSocket::with_options(options);
157
+ handshake.connect(engine_handshake).await?;
158
+ handshake
159
+ .send(ZmqMessage::from(encode_msgpack(&ready_message(
160
+ "HELLO", &config,
161
+ ))?))
162
+ .await?;
163
+
164
+ let init_frames = handshake.recv().await?.into_vec();
165
+ if init_frames.len() != 1 {
166
+ bail_unexpected_handshake_message!(
167
+ "expected one INIT frame from frontend, got {}",
168
+ init_frames.len()
169
+ );
170
+ }
171
+ let init: HandshakeInitMessage = decode_msgpack(init_frames[0].as_ref())?;
172
+
173
+ if init.addresses.inputs.is_empty() {
174
+ return Err(Error::UnexpectedHandshakeMessage {
175
+ message: "frontend INIT did not include an input address".to_string(),
176
+ });
177
+ }
178
+ if init.addresses.inputs.len() != init.addresses.outputs.len() {
179
+ return Err(Error::UnexpectedHandshakeMessage {
180
+ message: format!(
181
+ "frontend INIT input/output address count mismatch: {} inputs, {} outputs",
182
+ init.addresses.inputs.len(),
183
+ init.addresses.outputs.len()
184
+ ),
185
+ });
186
+ }
187
+
188
+ let mut data_sockets = Vec::with_capacity(init.addresses.inputs.len());
189
+ for (input_address, output_address) in
190
+ init.addresses.inputs.iter().zip(init.addresses.outputs.iter())
191
+ {
192
+ wait_for_ipc_endpoint(input_address, config.connect_timeout).await?;
193
+ wait_for_ipc_endpoint(output_address, config.connect_timeout).await?;
194
+
195
+ let mut input_options = SocketOptions::default();
196
+ input_options.peer_identity(peer_identity.clone());
197
+ let mut dealer = DealerSocket::with_options(input_options);
198
+ dealer.connect(input_address).await?;
199
+ dealer.send(ZmqMessage::from(ready_response_payload(&config)?)).await?;
200
+
201
+ let mut push = PushSocket::new();
202
+ push.connect(output_address).await?;
203
+
204
+ data_sockets.push(MockEngineDataSockets { dealer, push });
205
+ }
206
+
207
+ let coordinator = match (
208
+ init.addresses.coordinator_input.as_deref(),
209
+ init.addresses.coordinator_output.as_deref(),
210
+ ) {
211
+ (Some(coordinator_input), Some(coordinator_output)) => {
212
+ let mut input_sub = SubSocket::new();
213
+ input_sub.connect(coordinator_input).await?;
214
+ input_sub.subscribe("").await?;
215
+
216
+ let mut output_push = PushSocket::new();
217
+ output_push.connect(coordinator_output).await?;
218
+
219
+ let ready = input_sub.recv().await?.into_vec();
220
+ if ready.len() != 1 || ready[0].as_ref() != b"READY" {
221
+ bail_unexpected_handshake_message!(
222
+ "expected coordinator READY marker, got {:?}",
223
+ ready
224
+ );
225
+ }
226
+
227
+ Some(MockCoordinatorSockets {
228
+ input_sub,
229
+ output_push,
230
+ })
231
+ }
232
+ (None, None) => None,
233
+ _ => bail_unexpected_handshake_message!(
234
+ "coordinator handshake addresses must be both present or both absent"
235
+ ),
236
+ };
237
+
238
+ handshake
239
+ .send(ZmqMessage::from(encode_msgpack(&ready_message(
240
+ "READY", &config,
241
+ ))?))
242
+ .await?;
243
+
244
+ Ok(MockEngineSockets {
245
+ init,
246
+ data_sockets,
247
+ coordinator,
248
+ })
249
+ }
250
+
251
+ /// Join already-bootstrapped frontend input/output sockets directly.
252
+ pub async fn connect_to_bootstrapped_frontend(
253
+ input_address: impl AsRef<str>,
254
+ output_address: impl AsRef<str>,
255
+ engine_id: impl Into<EngineId>,
256
+ config: MockEngineConfig,
257
+ ) -> Result<(DealerSocket, PushSocket)> {
258
+ let input_address = input_address.as_ref();
259
+ let output_address = output_address.as_ref();
260
+ wait_for_ipc_endpoint(input_address, config.connect_timeout).await?;
261
+ wait_for_ipc_endpoint(output_address, config.connect_timeout).await?;
262
+
263
+ let peer_identity = peer_identity(engine_id)?;
264
+ let mut input_options = SocketOptions::default();
265
+ input_options.peer_identity(peer_identity);
266
+ let mut dealer = DealerSocket::with_options(input_options);
267
+ dealer.connect(input_address).await?;
268
+ dealer.send(ZmqMessage::from(ready_response_payload(&config)?)).await?;
269
+
270
+ let mut push = PushSocket::new();
271
+ push.connect(output_address).await?;
272
+
273
+ Ok((dealer, push))
274
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/dtype.rs ADDED
@@ -0,0 +1,40 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use serde::{Deserialize, Serialize};
2
+
3
+ /// Effective model dtype reported by the engine after config resolution.
4
+ #[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
5
+ pub enum ModelDtype {
6
+ #[serde(rename = "float16")]
7
+ Float16,
8
+ #[serde(rename = "bfloat16")]
9
+ BFloat16,
10
+ #[serde(rename = "float32")]
11
+ Float32,
12
+ }
13
+
14
+ impl ModelDtype {
15
+ pub fn as_str(self) -> &'static str {
16
+ match self {
17
+ Self::Float16 => "float16",
18
+ Self::BFloat16 => "bfloat16",
19
+ Self::Float32 => "float32",
20
+ }
21
+ }
22
+ }
23
+
24
+ #[cfg(test)]
25
+ mod tests {
26
+ use super::ModelDtype;
27
+
28
+ #[test]
29
+ fn serde_uses_protocol_dtype_strings() {
30
+ assert_eq!(
31
+ serde_json::to_value(ModelDtype::Float16).unwrap(),
32
+ serde_json::json!("float16")
33
+ );
34
+ assert_eq!(
35
+ serde_json::from_value::<ModelDtype>(serde_json::json!("bfloat16")).unwrap(),
36
+ ModelDtype::BFloat16
37
+ );
38
+ assert_eq!(ModelDtype::Float32.as_str(), "float32");
39
+ }
40
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/handshake.rs ADDED
@@ -0,0 +1,81 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::collections::BTreeMap;
2
+
3
+ use serde::{Deserialize, Serialize};
4
+
5
+ use crate::protocol::OpaqueValue;
6
+ use crate::protocol::dtype::ModelDtype;
7
+
8
+ /// Decoded engine startup-handshake payload sent on the handshake socket.
9
+ ///
10
+ /// Original Python payload construction:
11
+ /// <https://github.com/vllm-project/vllm/blob/c8d98f81f6/vllm/v1/engine/core.py#L1000-L1035>
12
+ #[derive(Debug, Clone, Default, Serialize, Deserialize)]
13
+ pub struct ReadyMessage {
14
+ #[serde(default)]
15
+ pub status: Option<String>,
16
+ #[serde(default)]
17
+ pub local: Option<bool>,
18
+ #[serde(default)]
19
+ pub headless: Option<bool>,
20
+ #[serde(default)]
21
+ pub parallel_config_hash: Option<String>,
22
+ }
23
+
24
+ /// Post-initialization configuration sent from each engine on the input socket
25
+ /// registration message, after the handshake completes.
26
+ ///
27
+ /// Contains values that may differ from the original config (e.g.
28
+ /// `max_model_len` after KV cache auto-fitting, `num_gpu_blocks` after
29
+ /// profiling).
30
+ ///
31
+ /// Original Python definition:
32
+ /// <https://github.com/vllm-project/vllm/blob/c9340e6f35/vllm/v1/engine/__init__.py#L68-L80>
33
+ #[derive(Debug, Clone, Serialize, Deserialize)]
34
+ pub struct EngineCoreReadyResponse {
35
+ /// Engine-reported maximum model context length (auto-fitted after
36
+ /// KV cache profiling and may differ from the original config value).
37
+ pub max_model_len: u64,
38
+ /// Number of GPU blocks available for KV cache on this engine.
39
+ pub num_gpu_blocks: u64,
40
+ /// KV cache block size (tokens per block).
41
+ pub block_size: u64,
42
+ /// DP coordinator stats publish address, if applicable.
43
+ pub dp_stats_address: Option<String>,
44
+ /// Effective model dtype after Python vLLM resolves `--dtype`.
45
+ pub dtype: ModelDtype,
46
+ /// Python vLLM version reported by the engine process.
47
+ pub vllm_version: String,
48
+ /// World size (TP * PP) from the parallel config.
49
+ pub world_size: u64,
50
+ /// Data parallelism size from the parallel config.
51
+ pub data_parallel_size: u64,
52
+ /// Total KV cache capacity in tokens, if reported.
53
+ pub kv_cache_size_tokens: Option<u64>,
54
+ /// Maximum achievable request concurrency given the KV cache, if reported.
55
+ pub kv_cache_max_concurrency: Option<f64>,
56
+ }
57
+
58
+ /// Frontend-owned ZMQ addresses that are sent to the engine during startup
59
+ /// handshake initialization.
60
+ ///
61
+ /// Original Python definition (`EngineZmqAddresses`):
62
+ /// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/utils.py#L53-L67>
63
+ #[derive(Debug, Clone, Serialize, Deserialize)]
64
+ pub struct HandshakeAddresses {
65
+ pub inputs: Vec<String>,
66
+ pub outputs: Vec<String>,
67
+ pub coordinator_input: Option<String>,
68
+ pub coordinator_output: Option<String>,
69
+ pub frontend_stats_publish_address: Option<String>,
70
+ }
71
+
72
+ /// Startup handshake payload sent from the frontend to initialize an engine
73
+ /// after receiving `HELLO`.
74
+ ///
75
+ /// Original Python definition (`EngineHandshakeMetadata`):
76
+ /// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/engine/utils.py#L69-L77>
77
+ #[derive(Debug, Clone, Serialize, Deserialize)]
78
+ pub struct HandshakeInitMessage {
79
+ pub addresses: HandshakeAddresses,
80
+ pub parallel_config: BTreeMap<String, OpaqueValue>,
81
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/logprobs.rs ADDED
@@ -0,0 +1,285 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ mod array;
2
+ #[cfg(test)]
3
+ mod tests;
4
+ mod wire;
5
+
6
+ use std::ops::{Deref, DerefMut};
7
+
8
+ use enum_as_inner::EnumAsInner;
9
+ use serde::{Deserialize, Deserializer, Serialize};
10
+
11
+ use self::wire::*;
12
+ use crate::error::{Error, Result, bail_ext_value_decode};
13
+ use crate::protocol::tensor::{WireArrayData, WireNdArray};
14
+
15
+ /// One token candidate and its logprob metadata for a single sequence position.
16
+ ///
17
+ /// The first entry in a [`PositionLogprobs`] is always the sampled/selected
18
+ /// token for that position. Any remaining entries follow the engine's returned
19
+ /// top-k candidate order.
20
+ #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
21
+ pub struct TokenLogprob {
22
+ pub token_id: u32,
23
+ pub logprob: f32,
24
+ /// The sampled/selected token uses its actual vocab rank. Remaining entries
25
+ /// use 1-based top-k ranks matching the engine's returned candidate
26
+ /// order.
27
+ pub rank: u32,
28
+ }
29
+
30
+ /// Logprob payload for one sequence position.
31
+ ///
32
+ /// This is the semantic Rust representation used by the public client API after
33
+ /// the lower-level ndarray/tensor wire payload has been decoded.
34
+ #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
35
+ pub struct PositionLogprobs {
36
+ pub entries: Vec<TokenLogprob>,
37
+ }
38
+
39
+ impl PositionLogprobs {
40
+ /// Convert one decoded logprobs row into this per-position form by grouping
41
+ /// each token/logprob pair together with the sampled/selected token's
42
+ /// actual vocab rank.
43
+ fn from_decoded_row(token_ids: &[u32], logprobs: &[f32], sampled_rank: u32) -> Result<Self> {
44
+ if token_ids.len() != logprobs.len() {
45
+ bail_ext_value_decode!(
46
+ "logprobs row length mismatch: token_ids={}, logprobs={}",
47
+ token_ids.len(),
48
+ logprobs.len()
49
+ );
50
+ }
51
+ if sampled_rank == 0 {
52
+ bail_ext_value_decode!("token_ranks must be >= 1 for decoded engine-core logprobs");
53
+ }
54
+
55
+ let mut entries = Vec::with_capacity(token_ids.len());
56
+ for (index, (&token_id, &logprob)) in token_ids.iter().zip(logprobs.iter()).enumerate() {
57
+ let rank = if index == 0 {
58
+ sampled_rank
59
+ } else {
60
+ index as u32
61
+ };
62
+ entries.push(TokenLogprob {
63
+ token_id,
64
+ logprob,
65
+ rank,
66
+ });
67
+ }
68
+ Ok(Self { entries })
69
+ }
70
+ }
71
+
72
+ /// Decoded per-request logprobs payload for one engine-core output.
73
+ ///
74
+ /// Unlike the Python wire payload, this public Rust type is already fully
75
+ /// semantic: one [`PositionLogprobs`] per scored position, each containing the
76
+ /// sampled/selected token plus any returned top-k alternatives for that same
77
+ /// position.
78
+ ///
79
+ /// The Python engine still sends logprobs as ndarray/tensor-shaped wire tuples.
80
+ /// Rust resolves that lower-level representation during decode and exposes only
81
+ /// this per-position form to callers.
82
+ #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
83
+ pub struct Logprobs {
84
+ /// One decoded logprobs record per scored position in this engine-core
85
+ /// output.
86
+ pub positions: Vec<PositionLogprobs>,
87
+ }
88
+
89
+ impl Logprobs {
90
+ /// Returns the number of scored positions in this payload.
91
+ pub fn len(&self) -> usize {
92
+ self.positions.len()
93
+ }
94
+
95
+ /// Returns whether the payload contains no scored positions.
96
+ pub fn is_empty(&self) -> bool {
97
+ self.positions.is_empty()
98
+ }
99
+ }
100
+
101
+ /// Output field wrapper that is initially deserialized from the Python wire
102
+ /// shape, then resolved into [`Logprobs`] before the decoded message is
103
+ /// returned to callers.
104
+ #[derive(Clone, PartialEq, Debug, EnumAsInner)]
105
+ pub enum MaybeWireLogprobs {
106
+ /// The logprobs are still in the wire format and need to be resolved by
107
+ /// looking up aux frames and decoding raw views. Should only be used
108
+ /// internally during deserialization.
109
+ Wire(Box<WireLogprobs>),
110
+ /// The actual decoded logprobs value,
111
+ Direct(Logprobs),
112
+ }
113
+
114
+ impl Deref for MaybeWireLogprobs {
115
+ type Target = Logprobs;
116
+
117
+ fn deref(&self) -> &Self::Target {
118
+ match self {
119
+ Self::Wire(_) => panic!("Logprobs is still in wire format"),
120
+ Self::Direct(value) => value,
121
+ }
122
+ }
123
+ }
124
+
125
+ impl DerefMut for MaybeWireLogprobs {
126
+ fn deref_mut(&mut self) -> &mut Self::Target {
127
+ match self {
128
+ Self::Wire(_) => panic!("Logprobs is still in wire format"),
129
+ Self::Direct(value) => value,
130
+ }
131
+ }
132
+ }
133
+
134
+ impl<'de> Deserialize<'de> for MaybeWireLogprobs {
135
+ fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
136
+ where
137
+ D: Deserializer<'de>,
138
+ {
139
+ // When deserializing, it's always in the wire form.
140
+ WireLogprobs::deserialize(deserializer).map(|v| Self::Wire(Box::new(v)))
141
+ }
142
+ }
143
+
144
+ impl Serialize for MaybeWireLogprobs {
145
+ fn serialize<S>(&self, serializer: S) -> std::result::Result<S::Ok, S::Error>
146
+ where
147
+ S: serde::Serializer,
148
+ {
149
+ // For testing purposes only. We don't actually serialize it into aux frames.
150
+ match self {
151
+ Self::Wire(value) => value.serialize(serializer),
152
+ Self::Direct(value) => WireLogprobs::from_direct(value)
153
+ .map_err(serde::ser::Error::custom)?
154
+ .serialize(serializer),
155
+ }
156
+ }
157
+ }
158
+
159
+ impl MaybeWireLogprobs {
160
+ /// Resolve the wire representation into decoded logprobs by looking up aux
161
+ /// frames and decoding raw views as needed.
162
+ pub(super) fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Self>
163
+ where
164
+ Frame: AsRef<[u8]>,
165
+ {
166
+ match self {
167
+ Self::Direct(value) => Ok(Self::Direct(value)),
168
+ Self::Wire(value) => value.resolve(frames, field_prefix).map(Self::Direct),
169
+ }
170
+ }
171
+ }
172
+
173
+ impl WireLogprobs {
174
+ /// Convert semantic per-position logprobs into the Python wire tuple shape.
175
+ ///
176
+ /// This exists mainly so Rust-side tests can inject semantic logprobs into
177
+ /// mocked engine-core outputs without manually building ndarray
178
+ /// raw-view tuples.
179
+ fn from_direct(value: &Logprobs) -> std::result::Result<Self, String> {
180
+ let rows = value.positions.len();
181
+ let cols = value.positions.first().map(|position| position.entries.len()).unwrap_or(0);
182
+
183
+ let mut token_ids = Vec::with_capacity(rows.saturating_mul(cols).saturating_mul(8));
184
+ let mut logprobs = Vec::with_capacity(rows.saturating_mul(cols).saturating_mul(4));
185
+ let mut token_ranks = Vec::with_capacity(rows.saturating_mul(8));
186
+
187
+ for (row_index, position) in value.positions.iter().enumerate() {
188
+ if position.entries.len() != cols {
189
+ return Err(format!(
190
+ "logprobs row {row_index} length mismatch: expected {cols}, got {}",
191
+ position.entries.len()
192
+ ));
193
+ }
194
+ let Some((sampled, _)) = position.entries.split_first() else {
195
+ return Err(format!("logprobs row {row_index} is empty"));
196
+ };
197
+
198
+ token_ranks.extend_from_slice(&(sampled.rank as i64).to_le_bytes());
199
+ for entry in &position.entries {
200
+ token_ids.extend_from_slice(&(entry.token_id as i64).to_le_bytes());
201
+ logprobs.extend_from_slice(&entry.logprob.to_le_bytes());
202
+ }
203
+ }
204
+
205
+ Ok(Self {
206
+ logprob_token_ids: WireNdArray {
207
+ dtype: "<i8".to_string(),
208
+ shape: vec![rows, cols],
209
+ data: WireArrayData::RawView(token_ids),
210
+ },
211
+ logprobs: WireNdArray {
212
+ dtype: "<f4".to_string(),
213
+ shape: vec![rows, cols],
214
+ data: WireArrayData::RawView(logprobs),
215
+ },
216
+ token_ranks: WireNdArray {
217
+ dtype: "<i8".to_string(),
218
+ shape: vec![rows],
219
+ data: WireArrayData::RawView(token_ranks),
220
+ },
221
+ cu_num_generated_tokens: None,
222
+ })
223
+ }
224
+
225
+ /// Resolve the wire-format logprobs into semantic [`Logprobs`] records by
226
+ /// looking up aux frames, decoding raw views, and grouping each row
227
+ /// into one [`PositionLogprobs`].
228
+ fn resolve<Frame>(self, frames: &[Frame], field_prefix: &str) -> Result<Logprobs>
229
+ where
230
+ Frame: AsRef<[u8]>,
231
+ {
232
+ if let Some(indices) = self.cu_num_generated_tokens {
233
+ bail_ext_value_decode!(
234
+ "{field_prefix}.cu_num_generated_tokens: \
235
+ expected None for per-request engine-core logprobs payload, got {indices:?}"
236
+ );
237
+ }
238
+
239
+ let token_ids = array::decode_array2_u32(
240
+ self.logprob_token_ids,
241
+ &format!("{field_prefix}.logprob_token_ids"),
242
+ frames,
243
+ )?;
244
+ let logprobs =
245
+ array::decode_array2_f32(self.logprobs, &format!("{field_prefix}.logprobs"), frames)?;
246
+ let token_ranks = array::decode_array1_u32(
247
+ self.token_ranks,
248
+ &format!("{field_prefix}.token_ranks"),
249
+ frames,
250
+ )?;
251
+
252
+ if token_ids.rows != logprobs.rows || token_ids.cols != logprobs.cols {
253
+ bail_ext_value_decode!(
254
+ "{field_prefix}: row shape mismatch between token ids ({}, {}) and logprobs ({}, {})",
255
+ token_ids.rows,
256
+ token_ids.cols,
257
+ logprobs.rows,
258
+ logprobs.cols
259
+ );
260
+ }
261
+ if token_ids.rows != token_ranks.len() {
262
+ bail_ext_value_decode!(
263
+ "{field_prefix}: token_ranks length {} does not match row count {}",
264
+ token_ranks.len(),
265
+ token_ids.rows
266
+ );
267
+ }
268
+
269
+ let mut positions = Vec::with_capacity(token_ids.rows);
270
+ for ((token_ids_row, logprobs_row), sampled_rank) in token_ids
271
+ .data
272
+ .chunks(token_ids.cols)
273
+ .zip(logprobs.data.chunks(logprobs.cols))
274
+ .zip(token_ranks)
275
+ {
276
+ positions.push(PositionLogprobs::from_decoded_row(
277
+ token_ids_row,
278
+ logprobs_row,
279
+ sampled_rank,
280
+ )?);
281
+ }
282
+
283
+ Ok(Logprobs { positions })
284
+ }
285
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/logprobs/array.rs ADDED
@@ -0,0 +1,306 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::io::Cursor;
2
+
3
+ use byteorder::{BigEndian, LittleEndian, NativeEndian, ReadBytesExt};
4
+ use itertools::Itertools as _;
5
+
6
+ use crate::error::{Error, Result, ext_value_decode};
7
+ use crate::protocol::tensor::{ShapeExt as _, WireArrayData, WireNdArray};
8
+
9
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
10
+ pub(super) enum ScalarType {
11
+ I32,
12
+ I64,
13
+ F32,
14
+ }
15
+
16
+ #[derive(Debug, Clone, Copy, PartialEq, Eq)]
17
+ pub(super) enum Endianness {
18
+ Little,
19
+ Big,
20
+ Native,
21
+ }
22
+
23
+ #[derive(Debug, Clone, PartialEq)]
24
+ pub(super) struct DecodedArray2<T> {
25
+ pub rows: usize,
26
+ pub cols: usize,
27
+ pub data: Vec<T>,
28
+ }
29
+
30
+ pub(super) fn decode_array2_u32<Frame>(
31
+ value: WireNdArray,
32
+ field: &str,
33
+ frames: &[Frame],
34
+ ) -> Result<DecodedArray2<u32>>
35
+ where
36
+ Frame: AsRef<[u8]>,
37
+ {
38
+ let (shape, bytes, scalar, endianness) =
39
+ decode_array_metadata(value, field, frames, &[ScalarType::I32, ScalarType::I64])?;
40
+ if shape.len() != 2 {
41
+ return Err(decode_error(
42
+ field,
43
+ &format!("expected rank-2 array, got rank {}", shape.len()),
44
+ ));
45
+ }
46
+
47
+ let data = match scalar {
48
+ ScalarType::I32 => decode_i32_vec(&bytes, endianness, field)?
49
+ .into_iter()
50
+ .map(|value| convert_to_u32(value, field))
51
+ .try_collect()?,
52
+ ScalarType::I64 => decode_i64_vec(&bytes, endianness, field)?
53
+ .into_iter()
54
+ .map(|value| convert_to_u32(value, field))
55
+ .try_collect()?,
56
+ ScalarType::F32 => unreachable!("scalar validation should reject f32"),
57
+ };
58
+ Ok(DecodedArray2 {
59
+ rows: shape[0],
60
+ cols: shape[1],
61
+ data,
62
+ })
63
+ }
64
+
65
+ pub(super) fn decode_array1_u32<Frame>(
66
+ value: WireNdArray,
67
+ field: &str,
68
+ frames: &[Frame],
69
+ ) -> Result<Vec<u32>>
70
+ where
71
+ Frame: AsRef<[u8]>,
72
+ {
73
+ let (shape, bytes, scalar, endianness) =
74
+ decode_array_metadata(value, field, frames, &[ScalarType::I32, ScalarType::I64])?;
75
+ if shape.len() != 1 {
76
+ return Err(decode_error(
77
+ field,
78
+ &format!("expected rank-1 array, got rank {}", shape.len()),
79
+ ));
80
+ }
81
+
82
+ let data = match scalar {
83
+ ScalarType::I32 => decode_i32_vec(&bytes, endianness, field)?
84
+ .into_iter()
85
+ .map(|value| convert_to_u32(value, field))
86
+ .try_collect()?,
87
+ ScalarType::I64 => decode_i64_vec(&bytes, endianness, field)?
88
+ .into_iter()
89
+ .map(|value| convert_to_u32(value, field))
90
+ .try_collect()?,
91
+ ScalarType::F32 => unreachable!("scalar validation should reject f32"),
92
+ };
93
+ Ok(data)
94
+ }
95
+
96
+ pub(super) fn decode_array2_f32<Frame>(
97
+ value: WireNdArray,
98
+ field: &str,
99
+ frames: &[Frame],
100
+ ) -> Result<DecodedArray2<f32>>
101
+ where
102
+ Frame: AsRef<[u8]>,
103
+ {
104
+ let (shape, bytes, _, endianness) =
105
+ decode_array_metadata(value, field, frames, &[ScalarType::F32])?;
106
+ if shape.len() != 2 {
107
+ return Err(decode_error(
108
+ field,
109
+ &format!("expected rank-2 array, got rank {}", shape.len()),
110
+ ));
111
+ }
112
+
113
+ let data = decode_f32_vec(&bytes, endianness, field)?;
114
+ Ok(DecodedArray2 {
115
+ rows: shape[0],
116
+ cols: shape[1],
117
+ data,
118
+ })
119
+ }
120
+
121
+ pub(super) fn decode_array_metadata<Frame>(
122
+ value: WireNdArray,
123
+ field: &str,
124
+ frames: &[Frame],
125
+ expected_scalars: &[ScalarType],
126
+ ) -> Result<(Vec<usize>, Vec<u8>, ScalarType, Endianness)>
127
+ where
128
+ Frame: AsRef<[u8]>,
129
+ {
130
+ let WireNdArray { dtype, shape, data } = value;
131
+ let (scalar, endianness) = parse_dtype(&dtype, field)?;
132
+ if !expected_scalars.contains(&scalar) {
133
+ return Err(decode_error(
134
+ field,
135
+ &format!("expected dtype in {:?}, got {}", expected_scalars, dtype),
136
+ ));
137
+ }
138
+
139
+ let bytes = resolve_array_bytes(data, field, frames)?;
140
+ validate_byte_length(shape.as_slice(), bytes.len(), field, scalar)?;
141
+ Ok((shape, bytes, scalar, endianness))
142
+ }
143
+
144
+ pub(super) fn parse_dtype(dtype: &str, field: &str) -> Result<(ScalarType, Endianness)> {
145
+ let (endianness, body) = match dtype.as_bytes().first().copied() {
146
+ Some(b'<') => (Endianness::Little, &dtype[1..]),
147
+ Some(b'>') => (Endianness::Big, &dtype[1..]),
148
+ Some(b'=') => (Endianness::Native, &dtype[1..]),
149
+ Some(b'|') => (Endianness::Native, &dtype[1..]),
150
+ _ => (Endianness::Native, dtype),
151
+ };
152
+
153
+ let scalar = match body {
154
+ "i4" | "int32" => ScalarType::I32,
155
+ "i8" | "int64" => ScalarType::I64,
156
+ "f4" | "float32" => ScalarType::F32,
157
+ _ => {
158
+ return Err(decode_error(
159
+ field,
160
+ &format!("unsupported dtype string {dtype:?}"),
161
+ ));
162
+ }
163
+ };
164
+ Ok((scalar, endianness))
165
+ }
166
+
167
+ pub(super) fn resolve_array_bytes<Frame>(
168
+ value: WireArrayData,
169
+ field: &str,
170
+ frames: &[Frame],
171
+ ) -> Result<Vec<u8>>
172
+ where
173
+ Frame: AsRef<[u8]>,
174
+ {
175
+ match value {
176
+ WireArrayData::RawView(bytes) => Ok(bytes),
177
+ WireArrayData::AuxIndex(index) => {
178
+ let frame = frames.get(index).ok_or_else(|| {
179
+ decode_error(
180
+ field,
181
+ &format!(
182
+ "aux frame index {index} out of range for {} frames",
183
+ frames.len()
184
+ ),
185
+ )
186
+ })?;
187
+ Ok(frame.as_ref().to_vec())
188
+ }
189
+ }
190
+ }
191
+
192
+ pub(super) fn validate_byte_length(
193
+ shape: &[usize],
194
+ byte_len: usize,
195
+ field: &str,
196
+ scalar: ScalarType,
197
+ ) -> Result<()> {
198
+ let element_count = shape
199
+ .checked_numel()
200
+ .ok_or_else(|| decode_error(field, "shape element count overflowed usize"))?;
201
+ let element_size = match scalar {
202
+ ScalarType::I32 | ScalarType::F32 => 4,
203
+ ScalarType::I64 => 8,
204
+ };
205
+ let expected = element_count
206
+ .checked_mul(element_size)
207
+ .ok_or_else(|| decode_error(field, "byte length overflowed usize"))?;
208
+ if expected != byte_len {
209
+ return Err(decode_error(
210
+ field,
211
+ &format!("byte length mismatch: expected {expected}, got {byte_len}"),
212
+ ));
213
+ }
214
+ Ok(())
215
+ }
216
+
217
+ pub(super) fn decode_i32_vec(
218
+ bytes: &[u8],
219
+ endianness: Endianness,
220
+ field: &str,
221
+ ) -> Result<Vec<i32>> {
222
+ if !bytes.len().is_multiple_of(4) {
223
+ return Err(decode_error(
224
+ field,
225
+ &format!("byte length {} is not divisible by 4", bytes.len()),
226
+ ));
227
+ }
228
+ let mut cursor = Cursor::new(bytes);
229
+ let mut values = Vec::with_capacity(bytes.len() / 4);
230
+ while (cursor.position() as usize) < bytes.len() {
231
+ let value = match endianness {
232
+ Endianness::Little => cursor.read_i32::<LittleEndian>(),
233
+ Endianness::Big => cursor.read_i32::<BigEndian>(),
234
+ Endianness::Native => cursor.read_i32::<NativeEndian>(),
235
+ }
236
+ .map_err(|error| decode_error(field, &format!("failed to read i32 payload: {error}")))?;
237
+ values.push(value);
238
+ }
239
+ Ok(values)
240
+ }
241
+
242
+ pub(super) fn decode_f32_vec(
243
+ bytes: &[u8],
244
+ endianness: Endianness,
245
+ field: &str,
246
+ ) -> Result<Vec<f32>> {
247
+ if !bytes.len().is_multiple_of(4) {
248
+ return Err(decode_error(
249
+ field,
250
+ &format!("byte length {} is not divisible by 4", bytes.len()),
251
+ ));
252
+ }
253
+ let mut cursor = Cursor::new(bytes);
254
+ let mut values = Vec::with_capacity(bytes.len() / 4);
255
+ while (cursor.position() as usize) < bytes.len() {
256
+ let value = match endianness {
257
+ Endianness::Little => cursor.read_f32::<LittleEndian>(),
258
+ Endianness::Big => cursor.read_f32::<BigEndian>(),
259
+ Endianness::Native => cursor.read_f32::<NativeEndian>(),
260
+ }
261
+ .map_err(|error| decode_error(field, &format!("failed to read f32 payload: {error}")))?;
262
+ values.push(value);
263
+ }
264
+ Ok(values)
265
+ }
266
+
267
+ pub(super) fn decode_i64_vec(
268
+ bytes: &[u8],
269
+ endianness: Endianness,
270
+ field: &str,
271
+ ) -> Result<Vec<i64>> {
272
+ if !bytes.len().is_multiple_of(8) {
273
+ return Err(decode_error(
274
+ field,
275
+ &format!("byte length {} is not divisible by 8", bytes.len()),
276
+ ));
277
+ }
278
+ let mut cursor = Cursor::new(bytes);
279
+ let mut values = Vec::with_capacity(bytes.len() / 8);
280
+ while (cursor.position() as usize) < bytes.len() {
281
+ let value = match endianness {
282
+ Endianness::Little => cursor.read_i64::<LittleEndian>(),
283
+ Endianness::Big => cursor.read_i64::<BigEndian>(),
284
+ Endianness::Native => cursor.read_i64::<NativeEndian>(),
285
+ }
286
+ .map_err(|error| decode_error(field, &format!("failed to read i64 payload: {error}")))?;
287
+ values.push(value);
288
+ }
289
+ Ok(values)
290
+ }
291
+
292
+ fn convert_to_u32<I>(value: I, field: &str) -> Result<u32>
293
+ where
294
+ I: TryInto<u32> + std::fmt::Display + Copy,
295
+ {
296
+ value.try_into().map_err(|_| {
297
+ decode_error(
298
+ field,
299
+ &format!("expected non-negative token id/rank that fits in u32, got {value}"),
300
+ )
301
+ })
302
+ }
303
+
304
+ pub(super) fn decode_error(field: &str, reason: &str) -> Error {
305
+ ext_value_decode!("{field}: {reason}")
306
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/logprobs/tests.rs ADDED
@@ -0,0 +1,302 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::collections::BTreeSet;
2
+
3
+ use bytes::Bytes;
4
+ use rmpv::Value;
5
+
6
+ use super::{Logprobs, PositionLogprobs, TokenLogprob};
7
+ use crate::protocol::output::{EngineCoreFinishReason, decode_engine_core_outputs};
8
+
9
+ fn encode_value(value: &Value) -> Vec<u8> {
10
+ let mut out = Vec::new();
11
+ rmpv::encode::write_value(&mut out, value).unwrap();
12
+ out
13
+ }
14
+
15
+ fn output_wire_with_custom_fields(
16
+ new_logprobs: Option<Value>,
17
+ prompt_logprobs: Option<Value>,
18
+ ) -> Value {
19
+ Value::Array(vec![
20
+ Value::from(0),
21
+ Value::Array(vec![Value::Array(vec![
22
+ Value::from("req-1"),
23
+ Value::Array(vec![Value::from(7), Value::from(8)]),
24
+ new_logprobs.unwrap_or(Value::Nil),
25
+ prompt_logprobs.unwrap_or(Value::Nil),
26
+ Value::Nil,
27
+ Value::from(EngineCoreFinishReason::Length as u8),
28
+ ])]),
29
+ Value::Nil,
30
+ Value::from(0.0),
31
+ Value::Nil,
32
+ Value::Array(vec![Value::from("req-1")]),
33
+ ])
34
+ }
35
+
36
+ fn ndarray_value(dtype: &str, shape: &[usize], data: Value) -> Value {
37
+ Value::Array(vec![
38
+ Value::from(dtype),
39
+ Value::Array(shape.iter().copied().map(Value::from).collect()),
40
+ data,
41
+ ])
42
+ }
43
+
44
+ fn inline_logprobs_value() -> Value {
45
+ let ids = Value::Ext(
46
+ 3,
47
+ vec![
48
+ 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0,
49
+ 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0,
50
+ ],
51
+ );
52
+ let probs = Value::Ext(
53
+ 3,
54
+ vec![
55
+ 0, 0, 128, 63, 0, 0, 0, 64, 0, 0, 64, 64, 0, 0, 128, 64, 0, 0, 160, 64, 0, 0, 192, 64,
56
+ ],
57
+ );
58
+ let ranks = Value::Ext(3, vec![1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0]);
59
+ Value::Array(vec![
60
+ ndarray_value("<i8", &[2, 3], ids),
61
+ ndarray_value("<f4", &[2, 3], probs),
62
+ ndarray_value("<i8", &[2], ranks),
63
+ Value::Nil,
64
+ ])
65
+ }
66
+
67
+ fn inline_prompt_logprobs_value() -> Value {
68
+ let ids = Value::Ext(
69
+ 3,
70
+ vec![
71
+ 10, 0, 0, 0, 0, 0, 0, 0, 11, 0, 0, 0, 0, 0, 0, 0, 12, 0, 0, 0, 0, 0, 0, 0, 13, 0, 0, 0,
72
+ 0, 0, 0, 0, 14, 0, 0, 0, 0, 0, 0, 0, 15, 0, 0, 0, 0, 0, 0, 0,
73
+ ],
74
+ );
75
+ let probs = Value::Ext(
76
+ 3,
77
+ vec![
78
+ 0, 0, 32, 65, 0, 0, 48, 65, 0, 0, 64, 65, 0, 0, 80, 65, 0, 0, 96, 65, 0, 0, 112, 65,
79
+ ],
80
+ );
81
+ let ranks = Value::Ext(3, vec![3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0, 0, 0, 0]);
82
+ Value::Array(vec![
83
+ ndarray_value("int64", &[2, 3], ids),
84
+ ndarray_value("float32", &[2, 3], probs),
85
+ ndarray_value("int64", &[2], ranks),
86
+ Value::Nil,
87
+ ])
88
+ }
89
+
90
+ fn expected_sample_logprobs() -> Logprobs {
91
+ Logprobs {
92
+ positions: vec![
93
+ PositionLogprobs {
94
+ entries: vec![
95
+ TokenLogprob {
96
+ token_id: 1,
97
+ logprob: 1.0,
98
+ rank: 1,
99
+ },
100
+ TokenLogprob {
101
+ token_id: 2,
102
+ logprob: 2.0,
103
+ rank: 1,
104
+ },
105
+ TokenLogprob {
106
+ token_id: 3,
107
+ logprob: 3.0,
108
+ rank: 2,
109
+ },
110
+ ],
111
+ },
112
+ PositionLogprobs {
113
+ entries: vec![
114
+ TokenLogprob {
115
+ token_id: 4,
116
+ logprob: 4.0,
117
+ rank: 2,
118
+ },
119
+ TokenLogprob {
120
+ token_id: 5,
121
+ logprob: 5.0,
122
+ rank: 1,
123
+ },
124
+ TokenLogprob {
125
+ token_id: 6,
126
+ logprob: 6.0,
127
+ rank: 2,
128
+ },
129
+ ],
130
+ },
131
+ ],
132
+ }
133
+ }
134
+
135
+ fn expected_prompt_logprobs() -> Logprobs {
136
+ Logprobs {
137
+ positions: vec![
138
+ PositionLogprobs {
139
+ entries: vec![
140
+ TokenLogprob {
141
+ token_id: 10,
142
+ logprob: 10.0,
143
+ rank: 3,
144
+ },
145
+ TokenLogprob {
146
+ token_id: 11,
147
+ logprob: 11.0,
148
+ rank: 1,
149
+ },
150
+ TokenLogprob {
151
+ token_id: 12,
152
+ logprob: 12.0,
153
+ rank: 2,
154
+ },
155
+ ],
156
+ },
157
+ PositionLogprobs {
158
+ entries: vec![
159
+ TokenLogprob {
160
+ token_id: 13,
161
+ logprob: 13.0,
162
+ rank: 4,
163
+ },
164
+ TokenLogprob {
165
+ token_id: 14,
166
+ logprob: 14.0,
167
+ rank: 1,
168
+ },
169
+ TokenLogprob {
170
+ token_id: 15,
171
+ logprob: 15.0,
172
+ rank: 2,
173
+ },
174
+ ],
175
+ },
176
+ ],
177
+ }
178
+ }
179
+
180
+ #[test]
181
+ fn decodes_inline_new_logprobs() {
182
+ let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
183
+ Some(inline_logprobs_value()),
184
+ None,
185
+ )))];
186
+ let decoded = decode_engine_core_outputs(&frames).unwrap().into_request_batch().unwrap();
187
+
188
+ let logprobs = decoded.outputs[0].new_logprobs.clone().unwrap().into_direct().unwrap();
189
+ assert_eq!(logprobs, expected_sample_logprobs());
190
+ assert_eq!(
191
+ decoded.finished_requests,
192
+ Some(BTreeSet::from(["req-1".to_string()]))
193
+ );
194
+ }
195
+
196
+ #[test]
197
+ fn decodes_multipart_new_logprobs() {
198
+ let frames = vec![
199
+ Bytes::from(encode_value(&output_wire_with_custom_fields(
200
+ Some(Value::Array(vec![
201
+ ndarray_value("<i8", &[2, 3], Value::from(1)),
202
+ ndarray_value("<f4", &[2, 3], Value::from(2)),
203
+ ndarray_value("<i8", &[2], Value::from(3)),
204
+ Value::Nil,
205
+ ])),
206
+ None,
207
+ ))),
208
+ Bytes::from_static(&[
209
+ 1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 4, 0, 0, 0, 0,
210
+ 0, 0, 0, 5, 0, 0, 0, 0, 0, 0, 0, 6, 0, 0, 0, 0, 0, 0, 0,
211
+ ]),
212
+ Bytes::from_static(&[
213
+ 0, 0, 128, 63, 0, 0, 0, 64, 0, 0, 64, 64, 0, 0, 128, 64, 0, 0, 160, 64, 0, 0, 192, 64,
214
+ ]),
215
+ Bytes::from_static(&[1, 0, 0, 0, 0, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 0]),
216
+ ];
217
+ let decoded = decode_engine_core_outputs(&frames).unwrap().into_request_batch().unwrap();
218
+
219
+ let logprobs = decoded.outputs[0].new_logprobs.clone().unwrap().into_direct().unwrap();
220
+ assert_eq!(logprobs, expected_sample_logprobs());
221
+ }
222
+
223
+ #[test]
224
+ fn decodes_inline_prompt_logprobs() {
225
+ let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
226
+ None,
227
+ Some(inline_prompt_logprobs_value()),
228
+ )))];
229
+ let decoded = decode_engine_core_outputs(&frames).unwrap().into_request_batch().unwrap();
230
+
231
+ let logprobs = decoded.outputs[0]
232
+ .new_prompt_logprobs_tensors
233
+ .clone()
234
+ .unwrap()
235
+ .into_direct()
236
+ .unwrap();
237
+ assert_eq!(logprobs, expected_prompt_logprobs());
238
+ }
239
+
240
+ #[test]
241
+ fn decodes_big_endian_payloads() {
242
+ let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
243
+ Some(Value::Array(vec![
244
+ ndarray_value(">i4", &[1, 2], Value::Ext(3, vec![0, 0, 0, 1, 0, 0, 0, 2])),
245
+ ndarray_value(
246
+ ">f4",
247
+ &[1, 2],
248
+ Value::Ext(3, vec![63, 128, 0, 0, 64, 0, 0, 0]),
249
+ ),
250
+ ndarray_value(">i4", &[1], Value::Ext(3, vec![0, 0, 0, 3])),
251
+ Value::Nil,
252
+ ])),
253
+ None,
254
+ )))];
255
+ let decoded = decode_engine_core_outputs(&frames).unwrap().into_request_batch().unwrap();
256
+ let logprobs = decoded.outputs[0].new_logprobs.clone().unwrap().into_direct().unwrap();
257
+ assert_eq!(
258
+ logprobs,
259
+ Logprobs {
260
+ positions: vec![PositionLogprobs {
261
+ entries: vec![
262
+ TokenLogprob {
263
+ token_id: 1,
264
+ logprob: 1.0,
265
+ rank: 3,
266
+ },
267
+ TokenLogprob {
268
+ token_id: 2,
269
+ logprob: 2.0,
270
+ rank: 1,
271
+ },
272
+ ],
273
+ }],
274
+ }
275
+ );
276
+ }
277
+
278
+ #[test]
279
+ fn rejects_non_none_cu_num_generated_tokens() {
280
+ let frames = vec![Bytes::from(encode_value(&output_wire_with_custom_fields(
281
+ Some(Value::Array(vec![
282
+ ndarray_value("<i8", &[1, 1], Value::Ext(3, vec![1, 0, 0, 0, 0, 0, 0, 0])),
283
+ ndarray_value("<f4", &[1, 1], Value::Ext(3, vec![0, 0, 128, 63])),
284
+ ndarray_value("<i8", &[1], Value::Ext(3, vec![1, 0, 0, 0, 0, 0, 0, 0])),
285
+ Value::Array(vec![Value::from(0usize), Value::from(1usize)]),
286
+ ])),
287
+ None,
288
+ )))];
289
+
290
+ let error = decode_engine_core_outputs(&frames).unwrap_err();
291
+ let crate::error::Error::ExtValueDecode { message } = &error else {
292
+ panic!("expected ValueDecodeExt");
293
+ };
294
+ assert_eq!(
295
+ message,
296
+ "new_logprobs.cu_num_generated_tokens: expected None for per-request engine-core logprobs payload, got [0, 1]"
297
+ );
298
+ assert_eq!(
299
+ error.to_string(),
300
+ "messagepack ext value decode failed: new_logprobs.cu_num_generated_tokens: expected None for per-request engine-core logprobs payload, got [0, 1]"
301
+ );
302
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/logprobs/wire.rs ADDED
@@ -0,0 +1,31 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use serde_tuple::{Deserialize_tuple, Serialize_tuple};
2
+
3
+ use crate::protocol::tensor::WireNdArray;
4
+
5
+ /// Python wire representation of `LogprobsLists` / `LogprobsTensors` before
6
+ /// aux-frame references and raw-view payloads are resolved.
7
+ ///
8
+ /// This mirrors the tuple shape emitted by Python engine-core so serde can
9
+ /// first deserialize the raw wire payload before the Rust client converts it
10
+ /// into semantic per-position logprobs records.
11
+ ///
12
+ /// Original Python definition:
13
+ /// <https://github.com/vllm-project/vllm/blob/f22d6e026798a74e6542a52ef776c054f2de572a/vllm/v1/outputs.py#L23-L56>
14
+ #[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
15
+ pub struct WireLogprobs {
16
+ /// Wire array with shape `[num_positions, max_num_logprobs + 1]`.
17
+ pub logprob_token_ids: WireNdArray,
18
+ /// Wire array with shape `[num_positions, max_num_logprobs + 1]`.
19
+ pub logprobs: WireNdArray,
20
+ /// Wire array with shape `[num_positions]`.
21
+ ///
22
+ /// Python uses the field name `sampled_token_ranks` for sample logprobs and
23
+ /// `selected_token_ranks` for prompt logprobs. Rust keeps one neutral field
24
+ /// because both payloads share the same wire representation.
25
+ pub token_ranks: WireNdArray,
26
+ /// Preserved only for wire compatibility with batch-level Python tensors.
27
+ /// Scheduler-sliced per-request outputs should emit `None` here, and
28
+ /// the semantic Rust decoder rejects any other value.
29
+ #[serde(default)]
30
+ pub cu_num_generated_tokens: Option<Vec<usize>>,
31
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/lora.rs ADDED
@@ -0,0 +1,42 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use serde_tuple::{Deserialize_tuple, Serialize_tuple};
2
+
3
+ use crate::protocol::OpaqueValue;
4
+
5
+ /// Request for a LoRA adapter.
6
+ ///
7
+ /// Mirrors Python `vllm.lora.request.LoRARequest`, which is a msgspec
8
+ /// `array_like=True` struct. Keep the field order aligned with Python.
9
+ #[derive(Debug, Clone, PartialEq, Serialize_tuple, Deserialize_tuple)]
10
+ pub struct LoraRequest {
11
+ pub lora_name: String,
12
+ pub lora_int_id: u64,
13
+ pub lora_path: String,
14
+ #[serde(default)]
15
+ pub base_model_name: Option<String>,
16
+ #[serde(default)]
17
+ pub tensorizer_config_dict: Option<OpaqueValue>,
18
+ #[serde(default)]
19
+ pub load_inplace: bool,
20
+ #[serde(default)]
21
+ pub is_3d_lora_weight: bool,
22
+ }
23
+
24
+ impl LoraRequest {
25
+ pub fn new(
26
+ lora_name: String,
27
+ lora_int_id: u64,
28
+ lora_path: String,
29
+ load_inplace: bool,
30
+ is_3d_lora_weight: bool,
31
+ ) -> Self {
32
+ Self {
33
+ lora_name,
34
+ lora_int_id,
35
+ lora_path,
36
+ base_model_name: None,
37
+ tensorizer_config_dict: None,
38
+ load_inplace,
39
+ is_3d_lora_weight,
40
+ }
41
+ }
42
+ }
asian_ai/performance/vLLM/rust/src/engine-core-client/src/protocol/mod.rs ADDED
@@ -0,0 +1,89 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ use std::any::type_name;
2
+ use std::io::Cursor;
3
+
4
+ use rmpv::Value;
5
+ use serde::{Deserialize, Serialize};
6
+ use thiserror_ext::AsReport;
7
+
8
+ use crate::error::{Error, Result};
9
+
10
+ /// Dynamic msgpack value used for schema positions that are preserved but not
11
+ /// yet strongly typed in the early-stage Rust client.
12
+ pub type OpaqueValue = Value;
13
+
14
+ fn default_opaque_value_nil() -> OpaqueValue {
15
+ Value::Nil
16
+ }
17
+
18
+ fn is_false(v: &bool) -> bool {
19
+ !v
20
+ }
21
+
22
+ pub mod dtype;
23
+ pub mod handshake;
24
+ pub mod logprobs;
25
+ pub mod lora;
26
+ pub mod multimodal;
27
+ pub mod output;
28
+ pub mod request;
29
+ pub mod sampling;
30
+ pub mod stats;
31
+ pub mod structured_outputs;
32
+ pub mod tensor;
33
+ pub mod utility;
34
+
35
+ /// Encode a Rust value into msgpack using the protocol crate's serde model.
36
+ pub fn encode_msgpack<T>(value: &T) -> Result<Vec<u8>>
37
+ where
38
+ T: Serialize + std::fmt::Debug,
39
+ {
40
+ rmp_serde::to_vec_named(value).map_err(|error| Error::Encode {
41
+ target_type: type_name::<T>(),
42
+ message: format!(
43
+ "failed to encode value `{:?}`: {}",
44
+ value,
45
+ error.to_report_string()
46
+ ),
47
+ })
48
+ }
49
+
50
+ /// Decode a msgpack payload into a strongly typed protocol value, with enhanced
51
+ /// error reporting.
52
+ pub fn decode_msgpack<T>(bytes: &[u8]) -> Result<T>
53
+ where
54
+ T: for<'de> Deserialize<'de>,
55
+ {
56
+ fn decode_value_preview(bytes: &[u8]) -> String {
57
+ match decode_value(bytes) {
58
+ Ok(value) => format!("{value}"),
59
+ Err(error) => format!("<value decode failed: {error}>"),
60
+ }
61
+ }
62
+
63
+ rmp_serde::from_slice(bytes).map_err(|error| Error::Decode {
64
+ target_type: type_name::<T>(),
65
+ message: format!("{error}; value fallback: {}", decode_value_preview(bytes)),
66
+ })
67
+ }
68
+
69
+ /// Decode a msgpack payload into a dynamic value for diagnostics and tests.
70
+ pub fn decode_value(bytes: &[u8]) -> Result<Value> {
71
+ Ok(rmpv::decode::read_value(&mut Cursor::new(bytes))?)
72
+ }
73
+
74
+ #[cfg(test)]
75
+ mod tests {
76
+ use std::collections::BTreeMap;
77
+
78
+ use super::*;
79
+
80
+ #[test]
81
+ fn decode_msgpack_includes_type_name_and_value_fallback() {
82
+ let error = decode_msgpack::<u64>(
83
+ &rmp_serde::to_vec_named(&BTreeMap::from([("status", "READY")])).unwrap(),
84
+ )
85
+ .unwrap_err();
86
+
87
+ expect_test::expect![[r#"messagepack decode failed for u64: wrong msgpack marker FixMap(1); value fallback: {"status": "READY"}"#]].assert_eq(&error.to_report_string());
88
+ }
89
+ }