luclima commited on
Commit
0090d22
·
verified ·
1 Parent(s): 2875c44

Upload folder using huggingface_hub (part 54)

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .gitattributes +3 -0
  2. asian_ai/performance/vLLM/tests/lora/test_moe_lora_ep_load.py +90 -0
  3. asian_ai/performance/vLLM/tests/lora/test_olmoe_tp.py +226 -0
  4. asian_ai/performance/vLLM/tests/lora/test_peft_helper.py +99 -0
  5. asian_ai/performance/vLLM/tests/lora/test_punica_ops.py +608 -0
  6. asian_ai/performance/vLLM/tests/lora/test_punica_ops_fp8.py +1001 -0
  7. asian_ai/performance/vLLM/tests/lora/test_punica_xpu_ops.py +298 -0
  8. asian_ai/performance/vLLM/tests/lora/test_quant_model.py +167 -0
  9. asian_ai/performance/vLLM/tests/lora/test_qwen35_densemodel_lora.py +370 -0
  10. asian_ai/performance/vLLM/tests/lora/test_qwen36_moe_lora.py +156 -0
  11. asian_ai/performance/vLLM/tests/lora/test_qwen3_unembed.py +100 -0
  12. asian_ai/performance/vLLM/tests/lora/test_qwen3_with_multi_loras.py +306 -0
  13. asian_ai/performance/vLLM/tests/lora/test_qwen3moe_tp.py +122 -0
  14. asian_ai/performance/vLLM/tests/lora/test_qwenvl.py +320 -0
  15. asian_ai/performance/vLLM/tests/lora/test_resolver.py +75 -0
  16. asian_ai/performance/vLLM/tests/lora/test_transformers_model.py +116 -0
  17. asian_ai/performance/vLLM/tests/lora/test_utils.py +201 -0
  18. asian_ai/performance/vLLM/tests/lora/test_whisper.py +134 -0
  19. asian_ai/performance/vLLM/tests/lora/test_worker.py +113 -0
  20. asian_ai/performance/vLLM/tests/lora/utils.py +410 -0
  21. asian_ai/performance/vLLM/tests/model_executor/__init__.py +0 -0
  22. asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_activations.py +234 -0
  23. asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_heads.py +481 -0
  24. asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_methods.py +499 -0
  25. asian_ai/performance/vLLM/tests/model_executor/layers/test_rocm_unquantized_gemm.py +91 -0
  26. asian_ai/performance/vLLM/tests/model_executor/model_loader/__init__.py +0 -0
  27. asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/__init__.py +0 -0
  28. asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/test_fastsafetensors_loader.py +28 -0
  29. asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/test_weight_utils.py +49 -0
  30. asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/__init__.py +0 -0
  31. asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/test_instanttensor_loader.py +28 -0
  32. asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/test_weight_utils.py +52 -0
  33. asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/__init__.py +0 -0
  34. asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/conftest.py +35 -0
  35. asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_loader.py +124 -0
  36. asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py +52 -0
  37. asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_utils.py +60 -0
  38. asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_weight_utils.py +66 -0
  39. asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/__init__.py +0 -0
  40. asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/conftest.py +92 -0
  41. asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py +560 -0
  42. asian_ai/performance/vLLM/tests/model_executor/model_loader/test_ep_weight_filter.py +361 -0
  43. asian_ai/performance/vLLM/tests/model_executor/model_loader/test_modelexpress_loader.py +133 -0
  44. asian_ai/performance/vLLM/tests/model_executor/model_loader/test_registry.py +90 -0
  45. asian_ai/performance/vLLM/tests/model_executor/model_loader/test_reload.py +498 -0
  46. asian_ai/performance/vLLM/tests/model_executor/model_loader/test_sharded_state_loader.py +165 -0
  47. asian_ai/performance/vLLM/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py +91 -0
  48. asian_ai/performance/vLLM/tests/model_executor/test_eagle_quantization.py +159 -0
  49. asian_ai/performance/vLLM/tests/model_executor/test_enabled_custom_ops.py +151 -0
  50. asian_ai/performance/vLLM/tests/model_executor/test_ernie45_vl_mrope.py +143 -0
.gitattributes CHANGED
@@ -880,3 +880,6 @@ asian_ai/performance/vLLM/docs/assets/features/disagg_prefill/overview.jpg filte
880
  asian_ai/performance/vLLM/docs/assets/training/layerwise.png filter=lfs diff=lfs merge=lfs -text
881
  asian_ai/performance/vLLM/docs/assets/training/layerwise_bad_loading.png filter=lfs diff=lfs merge=lfs -text
882
  asian_ai/performance/vLLM/docs/assets/training/layerwise_good_loading.png filter=lfs diff=lfs merge=lfs -text
 
 
 
 
880
  asian_ai/performance/vLLM/docs/assets/training/layerwise.png filter=lfs diff=lfs merge=lfs -text
881
  asian_ai/performance/vLLM/docs/assets/training/layerwise_bad_loading.png filter=lfs diff=lfs merge=lfs -text
882
  asian_ai/performance/vLLM/docs/assets/training/layerwise_good_loading.png filter=lfs diff=lfs merge=lfs -text
883
+ asian_ai/performance/vLLM/tests/multimodal/assets/rgba.png filter=lfs diff=lfs merge=lfs -text
884
+ asian_ai/performance/vLLM/tests/v1/ec_connector/integration/hato.jpg filter=lfs diff=lfs merge=lfs -text
885
+ asian_ai/performance/vLLM/tools/profiler/nsys_profile_tools/images/csv1.png filter=lfs diff=lfs merge=lfs -text
asian_ai/performance/vLLM/tests/lora/test_moe_lora_ep_load.py ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ """
4
+ End-to-end correctness test for 2D MoE LoRA expert-parallel
5
+ load-time slicing
6
+ """
7
+
8
+ import pytest
9
+ import torch
10
+
11
+ from vllm.lora.lora_model import LoRAModel, MoEEPLoadSpec
12
+ from vllm.lora.peft_helper import PEFTHelper
13
+
14
+ NUM_LAYERS = 48
15
+ GLOBAL_NUM_EXPERTS = 128
16
+ LOCAL_NUM_EXPERTS = 64 # ep_size = 2
17
+ EXPERT_PROJECTIONS = ("down_proj", "gate_proj", "up_proj")
18
+ NON_EXPERT_MODULES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate")
19
+
20
+
21
+ def _expected_lora_modules() -> set[str]:
22
+ """Replicate the set ``WorkerLoRAManager._load_adapter`` would build
23
+ from this model's ``packed_modules_mapping``."""
24
+ expected: set[str] = set(NON_EXPERT_MODULES)
25
+ for expert in range(GLOBAL_NUM_EXPERTS):
26
+ for proj in EXPERT_PROJECTIONS:
27
+ expected.add(f"experts.{expert}.{proj}")
28
+ return expected
29
+
30
+
31
+ def _load(lora_dir, peft_helper, *, moe_ep_spec, lora_id):
32
+ return LoRAModel.from_local_checkpoint(
33
+ lora_dir,
34
+ _expected_lora_modules(),
35
+ peft_helper=peft_helper,
36
+ lora_model_id=lora_id,
37
+ device="cpu",
38
+ moe_ep_spec=moe_ep_spec,
39
+ )
40
+
41
+
42
+ @pytest.mark.parametrize("ep_rank", [0, 1])
43
+ def test_moe_lora_ep2_real_qwen3moe(qwen3moe_lora_files, ep_rank):
44
+ """ep_size=2 against the real Qwen3-MoE adapter: each rank's loaded
45
+ LoRA has the right size, the right expert membership, and the
46
+ right tensor values."""
47
+ peft_helper = PEFTHelper.from_local_dir(
48
+ qwen3moe_lora_files, max_position_embeddings=4096
49
+ )
50
+
51
+ # Baseline: no spec → loads every expert × projection × layer plus
52
+ # all non-expert LoRA modules.
53
+ ground_truth = _load(qwen3moe_lora_files, peft_helper, moe_ep_spec=None, lora_id=1)
54
+ expected_baseline = (
55
+ GLOBAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
56
+ + len(NON_EXPERT_MODULES) * NUM_LAYERS
57
+ )
58
+ assert len(ground_truth.loras) == expected_baseline
59
+
60
+ # Sliced load: only this rank's experts; non-expert LoRA is untouched.
61
+ spec = MoEEPLoadSpec(
62
+ ep_rank=ep_rank,
63
+ local_num_experts=LOCAL_NUM_EXPERTS,
64
+ global_num_experts=GLOBAL_NUM_EXPERTS,
65
+ )
66
+ sliced = _load(
67
+ qwen3moe_lora_files,
68
+ peft_helper,
69
+ moe_ep_spec=spec,
70
+ lora_id=100 + ep_rank,
71
+ )
72
+
73
+ expected_sliced = (
74
+ LOCAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
75
+ + len(NON_EXPERT_MODULES) * NUM_LAYERS
76
+ )
77
+ assert len(sliced.loras) == expected_sliced
78
+
79
+ expert_start = ep_rank * LOCAL_NUM_EXPERTS
80
+ expert_end = expert_start + LOCAL_NUM_EXPERTS
81
+
82
+ for name, lora in sliced.loras.items():
83
+ gt = ground_truth.loras[name]
84
+ torch.testing.assert_close(lora.lora_a, gt.lora_a)
85
+ torch.testing.assert_close(lora.lora_b, gt.lora_b)
86
+ if ".experts." in name:
87
+ expert_idx = int(name.split(".experts.")[-1].split(".")[0])
88
+ assert expert_start <= expert_idx < expert_end, (
89
+ f"non-local expert {expert_idx} leaked: {name}"
90
+ )
asian_ai/performance/vLLM/tests/lora/test_olmoe_tp.py ADDED
@@ -0,0 +1,226 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+ import shutil
6
+ from collections.abc import Sequence
7
+
8
+ import pytest
9
+ import torch
10
+ from safetensors.torch import load_file, save_file
11
+
12
+ import vllm
13
+ from vllm.lora.request import LoRARequest
14
+ from vllm.platforms import current_platform
15
+
16
+ from ..utils import multi_gpu_test
17
+
18
+ MODEL_PATH = "allenai/OLMoE-1B-7B-0125-Instruct"
19
+
20
+ PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me. Do not return any additional explanation. Below is an instruction that describes a task, Write a response that appropriately completes the request.
21
+ "
22
+ ##Instruction:
23
+ candidate_poll contains tables such as candidate, people. Table candidate has columns such as Candidate_ID, People_ID, Poll_Source, Date, Support_rate, Consider_rate, Oppose_rate, Unsure_rate. Candidate_ID is the primary key.
24
+ Table people has columns such as People_ID, Sex, Name, Date_of_Birth, Height, Weight. People_ID is the primary key.
25
+ The People_ID of candidate is the foreign key of People_ID of people.
26
+
27
+
28
+ ###Input:
29
+ {context}
30
+
31
+ ###Response:""" # noqa: E501
32
+
33
+ EXPECTED_LORA_OUTPUT = [
34
+ "SELECT count(*) FROM candidate",
35
+ "SELECT count(*) FROM candidate",
36
+ "SELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1", # noqa: E501
37
+ "SELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1", # noqa: E501
38
+ ]
39
+
40
+ EXPECTED_BASE_MODEL_OUTPUT = [
41
+ "SELECT COUNT(Candidate_ID) FROM candidate",
42
+ "SELECT COUNT(Candidate_ID) FROM candidate",
43
+ "SELECT Candidate_ID, COUNT(*) as Total_Candidates\nFROM candidate\nINNER JOIN people ON candidate.People_ID = people.People_ID", # noqa: E501
44
+ # There are multiple acceptable responses
45
+ (
46
+ "SELECT Candidate_ID, Poll_Source FROM candidate WHERE People_ID IN (SELECT People_ID FROM people) ORDER BY COUNT(*) DESC LIMIT 1", # noqa: E501
47
+ "SELECT Candidate_ID, Poll_Source FROM candidate WHERE COUNT(People_ID) = (SELECT COUNT(People_ID) FROM people) ORDER BY Candidate_ID DESC LIMIT 1", # noqa: E501
48
+ ),
49
+ ]
50
+
51
+
52
+ def _output_matches(generated: str, accepted: str | Sequence[str]) -> bool:
53
+ if isinstance(accepted, str):
54
+ accepted = (accepted,)
55
+ return any(generated.startswith(s) for s in accepted)
56
+
57
+
58
+ def generate_and_test(
59
+ llm: vllm.LLM,
60
+ lora_path: str,
61
+ lora_id: list[int | None] | int | None,
62
+ compare_lower: bool = False,
63
+ ) -> None:
64
+ prompts = [
65
+ PROMPT_TEMPLATE.format(context="How many candidates are there?"),
66
+ PROMPT_TEMPLATE.format(context="Count the number of candidates."),
67
+ PROMPT_TEMPLATE.format(
68
+ context="Which poll resource provided the most number of candidate information?" # noqa: E501
69
+ ),
70
+ PROMPT_TEMPLATE.format(
71
+ context="Return the poll resource associated with the most candidates."
72
+ ),
73
+ ]
74
+
75
+ lora_request = None
76
+ if isinstance(lora_id, int):
77
+ lora_request = LoRARequest(str(lora_id), lora_id, lora_path)
78
+ elif isinstance(lora_id, list):
79
+ lora_request = [
80
+ LoRARequest(str(i), i, lora_path) if i is not None else None
81
+ for i in lora_id
82
+ ]
83
+
84
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=64)
85
+ outputs = llm.generate(prompts, sampling_params, lora_request=lora_request)
86
+ # Print the outputs.
87
+ generated_texts: list[str] = []
88
+ for output in outputs:
89
+ prompt = output.prompt
90
+ generated_text = output.outputs[0].text.strip()
91
+ generated_texts.append(generated_text)
92
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
93
+
94
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
95
+ req_lora_id = lora_id[i] if isinstance(lora_id, list) else lora_id
96
+ generated_text = generated_texts[i]
97
+ expected_output = (
98
+ EXPECTED_LORA_OUTPUT[i]
99
+ if req_lora_id is not None
100
+ else EXPECTED_BASE_MODEL_OUTPUT[i]
101
+ )
102
+
103
+ if compare_lower:
104
+ generated_text = generated_text.lower()
105
+ if isinstance(expected_output, str):
106
+ expected_output = (expected_output.lower(),)
107
+ else:
108
+ expected_output = tuple(s.lower() for s in expected_output)
109
+ assert _output_matches(generated_text, expected_output), (
110
+ f"Output {i}: {generated_text!r} does not match any of {expected_output!r}"
111
+ )
112
+
113
+
114
+ @pytest.mark.skipif(
115
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
116
+ )
117
+ def test_olmoe_lora(olmoe_lora_files, maybe_enable_lora_dual_stream):
118
+ # We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
119
+ # Otherwise, the lora-test will fail due to CUDA OOM.
120
+ llm = vllm.LLM(
121
+ MODEL_PATH,
122
+ max_model_len=1024,
123
+ enable_lora=True,
124
+ max_loras=4,
125
+ enforce_eager=True,
126
+ trust_remote_code=True,
127
+ enable_chunked_prefill=True,
128
+ )
129
+
130
+ generate_and_test(llm, olmoe_lora_files, lora_id=1)
131
+ generate_and_test(llm, olmoe_lora_files, lora_id=2)
132
+
133
+
134
+ def test_olmoe_lora_mixed(olmoe_lora_files):
135
+ llm = vllm.LLM(
136
+ MODEL_PATH,
137
+ max_model_len=1024,
138
+ enable_lora=True,
139
+ max_loras=4,
140
+ enforce_eager=True,
141
+ trust_remote_code=True,
142
+ enable_chunked_prefill=True,
143
+ )
144
+
145
+ generate_and_test(llm, olmoe_lora_files, lora_id=[1, None, 3, None])
146
+
147
+
148
+ def test_olmoe_lora_mixed_random(
149
+ olmoe_lora_files, tmp_path, maybe_enable_lora_dual_stream
150
+ ):
151
+ # Create a dummy LoRA with random weights based on the real one
152
+ random_lora_path = tmp_path / "random_lora"
153
+ shutil.copytree(olmoe_lora_files, random_lora_path)
154
+
155
+ weights_path = random_lora_path / "adapter_model.safetensors"
156
+ weights = load_file(str(weights_path))
157
+ random_weights = {k: torch.randn_like(v) for k, v in weights.items()}
158
+ save_file(random_weights, str(weights_path))
159
+
160
+ llm = vllm.LLM(
161
+ MODEL_PATH,
162
+ max_model_len=1024,
163
+ enable_lora=True,
164
+ max_loras=4,
165
+ enforce_eager=True,
166
+ trust_remote_code=True,
167
+ enable_chunked_prefill=True,
168
+ )
169
+
170
+ prompts = [
171
+ PROMPT_TEMPLATE.format(context="How many candidates are there?"),
172
+ PROMPT_TEMPLATE.format(context="Count the number of candidates."),
173
+ ]
174
+
175
+ lora_requests = [
176
+ LoRARequest("real", 1, olmoe_lora_files),
177
+ LoRARequest("random", 2, str(random_lora_path)),
178
+ ]
179
+
180
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=64)
181
+ outputs = llm.generate(prompts, sampling_params, lora_request=lora_requests)
182
+ assert outputs[0].outputs[0].text.strip().startswith(EXPECTED_LORA_OUTPUT[0])
183
+
184
+
185
+ @pytest.mark.skipif(
186
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
187
+ )
188
+ @pytest.mark.parametrize("fully_sharded_loras", [False, True])
189
+ @multi_gpu_test(num_gpus=2)
190
+ def test_olmoe_lora_tp2(olmoe_lora_files, fully_sharded_loras):
191
+ llm = vllm.LLM(
192
+ MODEL_PATH,
193
+ max_model_len=1024,
194
+ enable_lora=True,
195
+ max_loras=4,
196
+ enforce_eager=True,
197
+ trust_remote_code=True,
198
+ enable_chunked_prefill=True,
199
+ tensor_parallel_size=2,
200
+ fully_sharded_loras=fully_sharded_loras,
201
+ )
202
+
203
+ generate_and_test(llm, olmoe_lora_files, lora_id=1)
204
+ generate_and_test(llm, olmoe_lora_files, lora_id=2)
205
+
206
+
207
+ @pytest.mark.parametrize("fully_sharded_loras", [False, True])
208
+ @multi_gpu_test(num_gpus=4)
209
+ def test_olmoe_lora_tp4(olmoe_lora_files, fully_sharded_loras):
210
+ llm = vllm.LLM(
211
+ MODEL_PATH,
212
+ max_model_len=1024,
213
+ enable_lora=True,
214
+ max_loras=4,
215
+ enforce_eager=True,
216
+ trust_remote_code=True,
217
+ enable_chunked_prefill=True,
218
+ tensor_parallel_size=4,
219
+ fully_sharded_loras=fully_sharded_loras,
220
+ )
221
+ generate_and_test(
222
+ llm, olmoe_lora_files, lora_id=1, compare_lower=fully_sharded_loras
223
+ )
224
+ generate_and_test(
225
+ llm, olmoe_lora_files, lora_id=2, compare_lower=fully_sharded_loras
226
+ )
asian_ai/performance/vLLM/tests/lora/test_peft_helper.py ADDED
@@ -0,0 +1,99 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import json
5
+ import math
6
+ import shutil
7
+
8
+ import pytest
9
+
10
+ from vllm.config.lora import LoRAConfig
11
+ from vllm.lora.peft_helper import PEFTHelper
12
+
13
+ ERROR_CASES = [
14
+ (
15
+ "test_rank",
16
+ {"r": 1024},
17
+ "is greater than max_lora_rank",
18
+ ),
19
+ ("test_dora", {"use_dora": True}, "does not yet support DoRA"),
20
+ (
21
+ "test_modules_to_save",
22
+ {"modules_to_save": ["lm_head"]},
23
+ "only supports modules_to_save being None",
24
+ ),
25
+ ]
26
+
27
+
28
+ def test_peft_helper_pass(llama32_lora_files, tmp_path):
29
+ peft_helper = PEFTHelper.from_local_dir(
30
+ llama32_lora_files, max_position_embeddings=4096
31
+ )
32
+ lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
33
+ peft_helper.validate_legal(lora_config)
34
+ assert peft_helper.r == 8
35
+ assert peft_helper.lora_alpha == 32
36
+ target_modules = sorted(peft_helper.target_modules)
37
+
38
+ assert target_modules == [
39
+ "down_proj",
40
+ "embed_tokens",
41
+ "gate_proj",
42
+ "k_proj",
43
+ "lm_head",
44
+ "o_proj",
45
+ "q_proj",
46
+ "up_proj",
47
+ "v_proj",
48
+ ]
49
+ assert peft_helper.vllm_max_position_embeddings == 4096
50
+
51
+ # test RSLoRA
52
+ rslora_config = dict(use_rslora=True)
53
+ test_dir = tmp_path / "test_rslora"
54
+ shutil.copytree(llama32_lora_files, test_dir)
55
+
56
+ # Load and modify configuration
57
+ config_path = test_dir / "adapter_config.json"
58
+ with open(config_path) as f:
59
+ adapter_config = json.load(f)
60
+ # Apply configuration changes
61
+ adapter_config.update(rslora_config)
62
+
63
+ # Save modified configuration
64
+ with open(config_path, "w") as f:
65
+ json.dump(adapter_config, f)
66
+
67
+ peft_helper = PEFTHelper.from_local_dir(test_dir, max_position_embeddings=4096)
68
+ peft_helper.validate_legal(lora_config)
69
+ scaling = peft_helper.lora_alpha / math.sqrt(peft_helper.r)
70
+ assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
71
+
72
+
73
+ @pytest.mark.parametrize("test_name,config_change,expected_error", ERROR_CASES)
74
+ def test_peft_helper_error(
75
+ llama32_lora_files,
76
+ tmp_path,
77
+ test_name: str,
78
+ config_change: dict,
79
+ expected_error: str,
80
+ ):
81
+ test_dir = tmp_path / test_name
82
+ shutil.copytree(llama32_lora_files, test_dir)
83
+
84
+ # Load and modify configuration
85
+ config_path = test_dir / "adapter_config.json"
86
+ with open(config_path) as f:
87
+ adapter_config = json.load(f)
88
+ # Apply configuration changes
89
+ adapter_config.update(config_change)
90
+
91
+ # Save modified configuration
92
+ with open(config_path, "w") as f:
93
+ json.dump(adapter_config, f)
94
+ lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
95
+ # Test loading the adapter
96
+ with pytest.raises(ValueError, match=expected_error):
97
+ PEFTHelper.from_local_dir(
98
+ test_dir, max_position_embeddings=4096
99
+ ).validate_legal(lora_config)
asian_ai/performance/vLLM/tests/lora/test_punica_ops.py ADDED
@@ -0,0 +1,608 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ from threading import Lock
4
+
5
+ import pytest
6
+ import torch
7
+
8
+ import vllm.lora.ops.torch_ops as torch_ops
9
+ import vllm.lora.ops.triton_ops as triton_ops
10
+ from vllm.lora.ops.triton_ops import LoRAKernelMeta
11
+ from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
12
+ from vllm.platforms import current_platform
13
+ from vllm.utils.torch_utils import set_random_seed
14
+
15
+ from .utils import PunicaTensors, assert_close, generate_data_for_nslices
16
+
17
+ DEVICE_TYPE = current_platform.device_type
18
+
19
+
20
+ @pytest.fixture(autouse=True)
21
+ def reset_device(reset_default_device):
22
+ pass
23
+
24
+
25
+ # Utility shrink and expand operations used as reference implementations.
26
+ def sgmv_shrink_for_nslices(
27
+ nslices: int,
28
+ inputs_tensor: torch.Tensor,
29
+ lora_weights_lst: list[torch.Tensor],
30
+ out_tensor: torch.Tensor,
31
+ b_seq_start_loc: torch.Tensor,
32
+ seq_len_tensor: torch.Tensor,
33
+ prompt_lora_mapping: torch.Tensor,
34
+ batches: int,
35
+ max_seq_length: int,
36
+ num_tokens: int,
37
+ scaling: float,
38
+ ):
39
+ """
40
+ Wrapper around torch_ops.sgmv_shrink that handles any nslices.
41
+ """
42
+ for index in range(nslices):
43
+ torch_ops.sgmv_shrink(
44
+ inputs_tensor,
45
+ lora_weights_lst[index],
46
+ out_tensor[index],
47
+ b_seq_start_loc,
48
+ seq_len_tensor,
49
+ prompt_lora_mapping,
50
+ batches,
51
+ max_seq_length,
52
+ num_tokens,
53
+ scaling,
54
+ )
55
+
56
+
57
+ def sgmv_expand_for_nslices(
58
+ nslices: int,
59
+ hidden_size: int,
60
+ inputs_tensor: torch.Tensor,
61
+ lora_weights_lst: list[torch.Tensor],
62
+ out_tensor: torch.Tensor,
63
+ b_seq_start_loc: torch.Tensor,
64
+ seq_len_tensor: torch.Tensor,
65
+ prompt_lora_mapping: torch.Tensor,
66
+ batches: int,
67
+ max_seq_length: int,
68
+ num_tokens: int,
69
+ add_inputs: bool,
70
+ ) -> None:
71
+ """
72
+ Wrapper around torch_ops.sgmv_expand that handles any nslices.
73
+ """
74
+ if nslices == 1:
75
+ # Verify the torch's sgmv_expand op
76
+ torch_ops.sgmv_expand(
77
+ inputs_tensor[0],
78
+ lora_weights_lst[0],
79
+ out_tensor,
80
+ b_seq_start_loc,
81
+ seq_len_tensor,
82
+ prompt_lora_mapping,
83
+ batches,
84
+ max_seq_length,
85
+ num_tokens,
86
+ add_inputs=add_inputs,
87
+ )
88
+ else:
89
+ slice_offset = 0
90
+ for index in range(nslices):
91
+ lora_weights = lora_weights_lst[index]
92
+ torch_ops.sgmv_expand_slice(
93
+ inputs_tensor[index],
94
+ lora_weights,
95
+ out_tensor,
96
+ b_seq_start_loc,
97
+ seq_len_tensor,
98
+ prompt_lora_mapping,
99
+ batches,
100
+ max_seq_length,
101
+ num_tokens,
102
+ slice_offset,
103
+ hidden_size,
104
+ add_inputs=add_inputs,
105
+ )
106
+ slice_offset += hidden_size
107
+
108
+
109
+ _dict_lock = Lock()
110
+
111
+
112
+ def check_lora_shrink_kernel(
113
+ batches: int,
114
+ num_loras: int,
115
+ rank: int,
116
+ hidden_size: int,
117
+ nslices: int,
118
+ dtype: torch.dtype,
119
+ device: str,
120
+ seq_length: int,
121
+ scaling: float,
122
+ ):
123
+ """
124
+ Compare outputs of torch_ops.sgmv_shrink and triton_ops.lora_shrink
125
+ kernels.
126
+ """
127
+ data: PunicaTensors = generate_data_for_nslices(
128
+ batches,
129
+ hidden_size,
130
+ num_loras,
131
+ rank,
132
+ seq_length,
133
+ nslices,
134
+ dtype,
135
+ "shrink",
136
+ device,
137
+ )
138
+ max_seq_length, token_nums = data.meta()
139
+
140
+ # Setup metadata information for SGMV and reference kernels
141
+ sgmv_meta_args = (
142
+ data.b_seq_start_loc,
143
+ data.seq_len_tensor,
144
+ data.prompt_lora_mapping,
145
+ batches,
146
+ max_seq_length,
147
+ token_nums,
148
+ )
149
+
150
+ # Setup metadata information for the LoRA kernel.
151
+ lora_meta = LoRAKernelMeta.make(
152
+ max_loras=num_loras,
153
+ max_num_tokens=token_nums,
154
+ device=DEVICE_TYPE,
155
+ )
156
+ lora_meta.prepare_tensors(data.token_lora_mapping)
157
+
158
+ ref_out_tensor = data.ref_out_tensor
159
+ out_tensor = data.our_out_tensor.clone()
160
+
161
+ # Preventing cache error pointer.
162
+ with _dict_lock:
163
+ # lora_shrink kernel
164
+ _LORA_A_PTR_DICT.clear()
165
+ triton_ops.lora_shrink(
166
+ data.inputs_tensor,
167
+ data.lora_weights,
168
+ out_tensor,
169
+ *lora_meta.meta_args(token_nums=token_nums, specialize_active_lora=False),
170
+ scaling,
171
+ )
172
+
173
+ # Reference
174
+ sgmv_shrink_for_nslices(
175
+ nslices,
176
+ data.inputs_tensor,
177
+ data.lora_weights,
178
+ ref_out_tensor,
179
+ *sgmv_meta_args,
180
+ scaling,
181
+ )
182
+
183
+ assert_close(out_tensor, ref_out_tensor)
184
+
185
+
186
+ def check_lora_expand_kernel(
187
+ batches: int,
188
+ num_loras: int,
189
+ rank: int,
190
+ hidden_size: int,
191
+ nslices: int,
192
+ dtype: torch.dtype,
193
+ device: str,
194
+ seq_length: int,
195
+ add_inputs: bool,
196
+ ):
197
+ """
198
+ Compare outputs of torch_ops.sgmv_expand and triton_ops.lora_expand
199
+ kernels.
200
+ """
201
+ data: PunicaTensors = generate_data_for_nslices(
202
+ batches,
203
+ hidden_size,
204
+ num_loras,
205
+ rank,
206
+ seq_length,
207
+ nslices,
208
+ dtype,
209
+ "expand",
210
+ device,
211
+ )
212
+
213
+ max_seq_length, token_nums = data.meta()
214
+
215
+ # Setup metadata information for SGMV and reference kernels
216
+ sgmv_meta_args = (
217
+ data.b_seq_start_loc,
218
+ data.seq_len_tensor,
219
+ data.prompt_lora_mapping,
220
+ batches,
221
+ max_seq_length,
222
+ token_nums,
223
+ )
224
+
225
+ # Setup metadata information for the LoRA kernel.
226
+ lora_meta = LoRAKernelMeta.make(
227
+ max_loras=num_loras,
228
+ max_num_tokens=token_nums,
229
+ device=DEVICE_TYPE,
230
+ )
231
+ lora_meta.prepare_tensors(data.token_lora_mapping)
232
+
233
+ # Setup output tensors
234
+ ref_out_tensor = data.ref_out_tensor
235
+ out_tensor = data.our_out_tensor.clone()
236
+
237
+ with _dict_lock:
238
+ # lora_expand kernel
239
+ _LORA_B_PTR_DICT.clear()
240
+ triton_ops.lora_expand(
241
+ data.inputs_tensor,
242
+ data.lora_weights,
243
+ out_tensor,
244
+ *lora_meta.meta_args(token_nums=token_nums, specialize_active_lora=False),
245
+ offset_start=0,
246
+ add_inputs=add_inputs,
247
+ )
248
+
249
+ # Reference
250
+ sgmv_expand_for_nslices(
251
+ nslices,
252
+ hidden_size,
253
+ data.inputs_tensor,
254
+ data.lora_weights,
255
+ ref_out_tensor,
256
+ *sgmv_meta_args,
257
+ add_inputs=add_inputs,
258
+ )
259
+
260
+ assert_close(out_tensor, ref_out_tensor)
261
+
262
+
263
+ # Tests
264
+ # We test the punica kernels along 2 verticals mainly.
265
+ # 1. Variations in hidden_dim size
266
+ # 2. Variations in all other parameters like (batch_size, max_rank, num_loras
267
+ # etc.)
268
+
269
+ # We have collected the hidden_sizes included in the LoRA models
270
+ # currently supported by vLLM. It tests whether the corresponding Triton
271
+ # kernel can run normally when tensor parallelism is set to
272
+ # [1, 2, 4, 8, 16, 32, 64].
273
+ HIDDEN_SIZES = [
274
+ 128,
275
+ 256,
276
+ 512,
277
+ 896,
278
+ 1024,
279
+ 1152,
280
+ 1216,
281
+ 1280,
282
+ 1536,
283
+ 1664,
284
+ 2048,
285
+ 2240,
286
+ 2304,
287
+ 2368,
288
+ 2432,
289
+ 2560,
290
+ 2752,
291
+ 3072,
292
+ 3328,
293
+ 3456,
294
+ 3584,
295
+ 3712,
296
+ 4096,
297
+ 4480,
298
+ 4608,
299
+ 4736,
300
+ 4864,
301
+ 5120,
302
+ 5504,
303
+ 5632,
304
+ 5888,
305
+ 6144,
306
+ 6400,
307
+ 6848,
308
+ 6912,
309
+ 7168,
310
+ 7424,
311
+ 8192,
312
+ 8960,
313
+ 9216,
314
+ 9472,
315
+ 10240,
316
+ 11008,
317
+ 11264,
318
+ 13824,
319
+ 14336,
320
+ 14784,
321
+ 14848,
322
+ 15360,
323
+ 18944,
324
+ 22016,
325
+ 22528,
326
+ 24576,
327
+ 27392,
328
+ 27648,
329
+ 29568,
330
+ 29696,
331
+ 32000,
332
+ 32256,
333
+ 32512,
334
+ 32768,
335
+ 33024,
336
+ 36864,
337
+ 43264,
338
+ 49152,
339
+ 49408,
340
+ 60544,
341
+ 60672,
342
+ 64000,
343
+ 64256,
344
+ 102400,
345
+ 102656,
346
+ 128000,
347
+ 128256,
348
+ ]
349
+ # The size of TP
350
+ divisibility = [1, 2, 8, 16, 64]
351
+
352
+ all_hidden_size = []
353
+ for div in divisibility:
354
+ for hidden_size in HIDDEN_SIZES:
355
+ all_hidden_size.append(hidden_size // div)
356
+
357
+ HIDDEN_SIZES = list(set(all_hidden_size))
358
+
359
+ # Test params that focuses on hidden_size variation.
360
+ hs_test_params = {
361
+ "hidden_sizes": HIDDEN_SIZES,
362
+ "batches": [4],
363
+ "num_loras": [4],
364
+ "max_ranks": [32],
365
+ }
366
+
367
+ # General tests params that tests for variations in all dimensions
368
+ # except hidden_size.
369
+ test_params = {
370
+ "hidden_sizes": [2049],
371
+ "batches": [1, 4, 16, 32],
372
+ "num_loras": [1, 8, 32, 128],
373
+ "max_ranks": [1, 4, 8, 16, 32, 64, 128, 256],
374
+ }
375
+
376
+ DTYPES = [torch.float16, torch.bfloat16]
377
+ DEVICES = [f"{DEVICE_TYPE}:{0}"]
378
+ SEED = [0]
379
+
380
+
381
+ @pytest.mark.parametrize("batches", test_params["batches"])
382
+ @pytest.mark.parametrize("num_loras", test_params["num_loras"])
383
+ @pytest.mark.parametrize("rank", test_params["max_ranks"])
384
+ @pytest.mark.parametrize("hidden_size", test_params["hidden_sizes"])
385
+ @pytest.mark.parametrize("nslices", [1, 2, 3])
386
+ @pytest.mark.parametrize("dtype", DTYPES)
387
+ @pytest.mark.parametrize("device", DEVICES)
388
+ @pytest.mark.parametrize("seed", SEED)
389
+ @pytest.mark.parametrize("op_type", ["shrink", "expand"])
390
+ def test_kernels(
391
+ batches: int,
392
+ num_loras: int,
393
+ rank: int,
394
+ hidden_size: int,
395
+ nslices: int,
396
+ dtype: torch.dtype,
397
+ device: str,
398
+ seed: int,
399
+ op_type: str,
400
+ ):
401
+ """
402
+ Tests LoRA kernels.
403
+ """
404
+ torch.set_default_device(device)
405
+ torch.accelerator.set_device_index(device)
406
+ set_random_seed(seed)
407
+
408
+ if op_type == "shrink":
409
+ check_lora_shrink_kernel(
410
+ batches=batches,
411
+ num_loras=num_loras,
412
+ rank=rank,
413
+ hidden_size=hidden_size,
414
+ nslices=nslices,
415
+ dtype=dtype,
416
+ device=device,
417
+ seq_length=128,
418
+ scaling=0.5,
419
+ )
420
+ else:
421
+ check_lora_expand_kernel(
422
+ batches=batches,
423
+ num_loras=num_loras,
424
+ rank=rank,
425
+ hidden_size=hidden_size,
426
+ nslices=nslices,
427
+ dtype=dtype,
428
+ device=device,
429
+ seq_length=128,
430
+ add_inputs=True,
431
+ )
432
+
433
+
434
+ @pytest.mark.parametrize("batches", hs_test_params["batches"])
435
+ @pytest.mark.parametrize("num_loras", hs_test_params["num_loras"])
436
+ @pytest.mark.parametrize("rank", hs_test_params["max_ranks"])
437
+ @pytest.mark.parametrize("hidden_size", hs_test_params["hidden_sizes"])
438
+ @pytest.mark.parametrize("nslices", [1, 2, 3])
439
+ @pytest.mark.parametrize("dtype", DTYPES)
440
+ @pytest.mark.parametrize("device", DEVICES)
441
+ @pytest.mark.parametrize("seed", SEED)
442
+ @pytest.mark.parametrize("op_type", ["shrink", "expand"])
443
+ def test_kernels_hidden_size(
444
+ batches: int,
445
+ num_loras: int,
446
+ rank: int,
447
+ hidden_size: int,
448
+ nslices: int,
449
+ dtype: torch.dtype,
450
+ device: str,
451
+ seed: int,
452
+ op_type: str,
453
+ ):
454
+ """
455
+ Tests SGMV and LoRA kernels.
456
+ """
457
+ torch.set_default_device(device)
458
+ torch.accelerator.set_device_index(device)
459
+ set_random_seed(seed)
460
+
461
+ if op_type == "shrink":
462
+ check_lora_shrink_kernel(
463
+ batches=batches,
464
+ num_loras=num_loras,
465
+ rank=rank,
466
+ hidden_size=hidden_size,
467
+ nslices=nslices,
468
+ dtype=dtype,
469
+ device=device,
470
+ seq_length=128,
471
+ scaling=0.5,
472
+ )
473
+ else:
474
+ check_lora_expand_kernel(
475
+ batches=batches,
476
+ num_loras=num_loras,
477
+ rank=rank,
478
+ hidden_size=hidden_size,
479
+ nslices=nslices,
480
+ dtype=dtype,
481
+ device=device,
482
+ seq_length=128,
483
+ add_inputs=True,
484
+ )
485
+
486
+
487
+ @pytest.mark.parametrize("device", DEVICES)
488
+ def test_add_lora_fused_moe_early_exit(device):
489
+ """
490
+ Ensures add_lora_fused_moe does not invoke the LoRA kernel or
491
+ modify the output tensor when no_lora_flag_cpu is True
492
+ """
493
+ from types import SimpleNamespace
494
+
495
+ from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
496
+
497
+ torch.set_default_device(device)
498
+ torch.accelerator.set_device_index(device)
499
+
500
+ max_loras, num_tokens = 4, 16
501
+ num_experts, top_k, max_lora_rank = 8, 2, 16
502
+ K, N = 256, 128
503
+
504
+ # build PunicaWrapperGPU with minimal lora_config mock
505
+ lora_config = SimpleNamespace(
506
+ max_loras=max_loras,
507
+ specialize_active_lora=False,
508
+ )
509
+ wrapper = PunicaWrapperGPU(
510
+ max_num_batched_tokens=num_tokens,
511
+ max_batches=num_tokens,
512
+ device=device,
513
+ lora_config=lora_config,
514
+ )
515
+
516
+ # simulate a prior LoRA batch so the internal mapping is
517
+ # populated with stale LoRA IDs
518
+ lora_mapping = torch.zeros(
519
+ num_tokens,
520
+ dtype=torch.int32,
521
+ device=device,
522
+ )
523
+ lora_mapping[:8] = 1
524
+ lora_mapping[8:] = 2
525
+ wrapper.token_mapping_meta.prepare_tensors(lora_mapping)
526
+
527
+ # simulate a base-model batch (all -1)
528
+ base_mapping = torch.full(
529
+ (num_tokens,),
530
+ -1,
531
+ dtype=torch.int32,
532
+ device=device,
533
+ )
534
+ wrapper.token_mapping_meta.prepare_tensors(base_mapping)
535
+
536
+ assert wrapper.token_mapping_meta.no_lora_flag_cpu[0].item() is True
537
+
538
+ # dummy tensors for add_lora_fused_moe
539
+ y = torch.rand(num_tokens, top_k, N, dtype=torch.bfloat16, device=device)
540
+ y_snapshot = y.clone()
541
+ x = torch.rand(num_tokens, K, dtype=torch.bfloat16, device=device)
542
+
543
+ lora_a_stacked = (
544
+ torch.rand(
545
+ max_loras,
546
+ num_experts,
547
+ max_lora_rank,
548
+ K,
549
+ dtype=torch.bfloat16,
550
+ device=device,
551
+ ),
552
+ )
553
+ lora_b_stacked = (
554
+ torch.rand(
555
+ max_loras,
556
+ num_experts,
557
+ N,
558
+ max_lora_rank,
559
+ dtype=torch.bfloat16,
560
+ device=device,
561
+ ),
562
+ )
563
+ topk_weights = torch.ones(
564
+ num_tokens,
565
+ top_k,
566
+ dtype=torch.float32,
567
+ device=device,
568
+ )
569
+ adapter_enabled = torch.ones(
570
+ max_loras + 1,
571
+ dtype=torch.int32,
572
+ device=device,
573
+ )
574
+ shrink_config = expand_config = {
575
+ "BLOCK_SIZE_M": 16,
576
+ "BLOCK_SIZE_N": 32,
577
+ "BLOCK_SIZE_K": 64,
578
+ "GROUP_SIZE_M": 1,
579
+ "NUM_WARPS": 4,
580
+ "NUM_STAGES": 3,
581
+ "SPLIT_K": 1,
582
+ }
583
+
584
+ # call add_lora_fused_moe - the early exit should prevent any
585
+ # modification to the output
586
+ wrapper.add_lora_fused_moe(
587
+ y=y,
588
+ x=x,
589
+ lora_a_stacked=lora_a_stacked,
590
+ lora_b_stacked=lora_b_stacked,
591
+ topk_weights=topk_weights,
592
+ sorted_token_ids=None,
593
+ expert_ids=torch.zeros(
594
+ num_tokens * top_k,
595
+ dtype=torch.int32,
596
+ device=device,
597
+ ),
598
+ num_tokens_post_padded=None,
599
+ max_lora_rank=max_lora_rank,
600
+ top_k_num=top_k,
601
+ shrink_config=shrink_config,
602
+ expand_config=expand_config,
603
+ adapter_enabled=adapter_enabled,
604
+ )
605
+
606
+ assert torch.equal(y, y_snapshot), (
607
+ "add_lora_fused_moe modified output tensor despite no_lora_flag_cpu=True"
608
+ )
asian_ai/performance/vLLM/tests/lora/test_punica_ops_fp8.py ADDED
@@ -0,0 +1,1001 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ """FP8 accuracy tests for LoRA shrink and expand kernels.
4
+
5
+ Tests the FP8 kernels by:
6
+ 1. Quantizing bf16 inputs/weights to FP8
7
+ 2. Dequantizing them back to bf16
8
+ 3. Running the bf16 reference (sgmv_shrink/sgmv_expand) with dequantized values
9
+ 4. Comparing FP8 kernel output against this dequantized reference
10
+
11
+ This isolates kernel correctness from quantization precision loss,
12
+ allowing much tighter tolerances than comparing against the original bf16.
13
+ """
14
+
15
+ import math
16
+ from threading import Lock
17
+
18
+ import pytest
19
+ import torch
20
+
21
+ import vllm.lora.ops.torch_ops as torch_ops
22
+ import vllm.lora.ops.triton_ops as triton_ops
23
+ from vllm.lora.ops.triton_ops import LoRAKernelMeta
24
+ from vllm.lora.ops.triton_ops.lora_expand_fp8_op import (
25
+ _EXPAND_LORA_SCALE_PTR_DICT,
26
+ )
27
+ from vllm.lora.ops.triton_ops.lora_shrink_fp8_op import (
28
+ _SHRINK_LORA_SCALE_PTR_DICT,
29
+ )
30
+ from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
31
+ from vllm.platforms import current_platform
32
+ from vllm.utils.torch_utils import set_random_seed
33
+
34
+ DEVICE_TYPE = current_platform.device_type
35
+ DEVICES = [f"{DEVICE_TYPE}:{0}"]
36
+ SEED = [0]
37
+
38
+ _dict_lock = Lock()
39
+
40
+
41
+ @pytest.fixture(autouse=True)
42
+ def reset_device(reset_default_device):
43
+ pass
44
+
45
+
46
+ # ============================================================================
47
+ # Reference implementations (bf16 baseline)
48
+ # ============================================================================
49
+
50
+
51
+ def sgmv_shrink_for_nslices(
52
+ nslices,
53
+ inputs_tensor,
54
+ lora_weights_lst,
55
+ out_tensor,
56
+ b_seq_start_loc,
57
+ seq_len_tensor,
58
+ prompt_lora_mapping,
59
+ batches,
60
+ max_seq_length,
61
+ num_tokens,
62
+ scaling,
63
+ ):
64
+ """Wrapper around torch_ops.sgmv_shrink that handles any nslices."""
65
+ for index in range(nslices):
66
+ torch_ops.sgmv_shrink(
67
+ inputs_tensor,
68
+ lora_weights_lst[index],
69
+ out_tensor[index],
70
+ b_seq_start_loc,
71
+ seq_len_tensor,
72
+ prompt_lora_mapping,
73
+ batches,
74
+ max_seq_length,
75
+ num_tokens,
76
+ scaling,
77
+ )
78
+
79
+
80
+ def sgmv_expand_for_nslices(
81
+ nslices,
82
+ hidden_size,
83
+ inputs_tensor,
84
+ lora_weights_lst,
85
+ out_tensor,
86
+ b_seq_start_loc,
87
+ seq_len_tensor,
88
+ prompt_lora_mapping,
89
+ batches,
90
+ max_seq_length,
91
+ num_tokens,
92
+ add_inputs,
93
+ ):
94
+ """Wrapper around torch_ops.sgmv_expand that handles any nslices."""
95
+ if nslices == 1:
96
+ torch_ops.sgmv_expand(
97
+ inputs_tensor[0],
98
+ lora_weights_lst[0],
99
+ out_tensor,
100
+ b_seq_start_loc,
101
+ seq_len_tensor,
102
+ prompt_lora_mapping,
103
+ batches,
104
+ max_seq_length,
105
+ num_tokens,
106
+ add_inputs=add_inputs,
107
+ )
108
+ else:
109
+ slice_offset = 0
110
+ for index in range(nslices):
111
+ torch_ops.sgmv_expand_slice(
112
+ inputs_tensor[index],
113
+ lora_weights_lst[index],
114
+ out_tensor,
115
+ b_seq_start_loc,
116
+ seq_len_tensor,
117
+ prompt_lora_mapping,
118
+ batches,
119
+ max_seq_length,
120
+ num_tokens,
121
+ slice_offset,
122
+ hidden_size,
123
+ add_inputs=add_inputs,
124
+ )
125
+ slice_offset += hidden_size
126
+
127
+
128
+ # ============================================================================
129
+ # FP8 Quantization Helpers
130
+ # ============================================================================
131
+
132
+ FP8_DTYPE = torch.float8_e4m3fn
133
+ FP8_MAX = torch.finfo(FP8_DTYPE).max
134
+ FP8_MIN = torch.finfo(FP8_DTYPE).min
135
+
136
+
137
+ def quantize_to_fp8_per_tensor(
138
+ tensor: torch.Tensor,
139
+ ) -> tuple[torch.Tensor, torch.Tensor]:
140
+ """Quantize a tensor to FP8 with per-tensor scaling."""
141
+ amax = tensor.abs().float().max().clamp(min=1e-12)
142
+ scale = (amax / FP8_MAX).to(torch.float32)
143
+ fp8_tensor = (tensor.float() / scale).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
144
+ return fp8_tensor, scale.reshape(1)
145
+
146
+
147
+ def quantize_to_fp8_per_channel(
148
+ tensor: torch.Tensor,
149
+ channel_dim: int = 0,
150
+ ) -> tuple[torch.Tensor, torch.Tensor]:
151
+ """Quantize a tensor to FP8 with per-channel scaling.
152
+
153
+ For shrink lora_a weights of shape (num_loras, rank, hidden_size):
154
+ channel_dim=1 gives per-rank scaling -> scale shape (num_loras, rank)
155
+ For expand lora_b weights of shape (num_loras, hidden_size, rank):
156
+ channel_dim=1 gives per-hidden scaling -> scale shape (num_loras, hidden_size)
157
+ """
158
+ # Compute amax along all dims except the leading dims up to channel_dim+1
159
+ reduce_dims = list(range(channel_dim + 1, tensor.ndim))
160
+ if reduce_dims:
161
+ amax = tensor.abs().float().amax(dim=reduce_dims).clamp(min=1e-12)
162
+ else:
163
+ amax = tensor.abs().float().clamp(min=1e-12)
164
+ scale = (amax / FP8_MAX).to(torch.float32)
165
+
166
+ # Expand scale for broadcasting
167
+ for _ in reduce_dims:
168
+ scale = scale.unsqueeze(-1)
169
+ fp8_tensor = (tensor.float() / scale).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
170
+ scale = scale.squeeze()
171
+ if scale.ndim == 0:
172
+ scale = scale.unsqueeze(0)
173
+ return fp8_tensor, scale
174
+
175
+
176
+ def quantize_to_fp8_per_token(
177
+ tensor: torch.Tensor,
178
+ ) -> tuple[torch.Tensor, torch.Tensor]:
179
+ """Quantize a 2D tensor to FP8 with per-token (per-row) scaling.
180
+
181
+ Input shape: (num_tokens, hidden_size)
182
+ Returns: (fp8_tensor, scale) where scale shape is (num_tokens, 1)
183
+ """
184
+ assert tensor.ndim == 2
185
+ amax = tensor.abs().float().amax(dim=1, keepdim=True).clamp(min=1e-12)
186
+ scale = (amax / FP8_MAX).to(torch.float32)
187
+ fp8_tensor = (tensor.float() / scale).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
188
+ return fp8_tensor, scale
189
+
190
+
191
+ def quantize_to_fp8_blockwise(
192
+ tensor: torch.Tensor,
193
+ group_n: int,
194
+ group_k: int,
195
+ ) -> tuple[torch.Tensor, torch.Tensor]:
196
+ """Quantize a 2D or 3D tensor to FP8 with block-wise scaling.
197
+
198
+ For a 2D tensor (num_tokens, hidden_size):
199
+ Blocks of size (1, group_k) ->
200
+ scale shape (num_tokens, ceil(hidden_size/group_k))
201
+
202
+ For a 3D tensor (num_loras, N, K):
203
+ Blocks of size (group_n, group_k) ->
204
+ scale shape (num_loras, ceil(N/group_n), ceil(K/group_k))
205
+ """
206
+ if tensor.ndim == 2:
207
+ M, K = tensor.shape
208
+ n_blocks_k = math.ceil(K / group_k)
209
+ scale = torch.zeros(M, n_blocks_k, dtype=torch.float32, device=tensor.device)
210
+ fp8_tensor = torch.zeros_like(tensor, dtype=FP8_DTYPE)
211
+ for m in range(M):
212
+ for bk in range(n_blocks_k):
213
+ k_start = bk * group_k
214
+ k_end = min(k_start + group_k, K)
215
+ block = tensor[m, k_start:k_end].float()
216
+ amax = block.abs().max().clamp(min=1e-12)
217
+ s = (amax / FP8_MAX).to(torch.float32)
218
+ scale[m, bk] = s
219
+ fp8_tensor[m, k_start:k_end] = (
220
+ (block / s).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
221
+ )
222
+ return fp8_tensor, scale
223
+ elif tensor.ndim == 3:
224
+ L, N, K = tensor.shape
225
+ n_blocks_n = math.ceil(N / group_n)
226
+ n_blocks_k = math.ceil(K / group_k)
227
+ scale = torch.zeros(
228
+ L, n_blocks_n, n_blocks_k, dtype=torch.float32, device=tensor.device
229
+ )
230
+ fp8_tensor = torch.zeros_like(tensor, dtype=FP8_DTYPE)
231
+ for li in range(L):
232
+ for bn in range(n_blocks_n):
233
+ for bk in range(n_blocks_k):
234
+ n_start = bn * group_n
235
+ n_end = min(n_start + group_n, N)
236
+ k_start = bk * group_k
237
+ k_end = min(k_start + group_k, K)
238
+ block = tensor[li, n_start:n_end, k_start:k_end].float()
239
+ amax = block.abs().max().clamp(min=1e-12)
240
+ s = (amax / FP8_MAX).to(torch.float32)
241
+ scale[li, bn, bk] = s
242
+ fp8_tensor[li, n_start:n_end, k_start:k_end] = (
243
+ (block / s).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
244
+ )
245
+ return fp8_tensor, scale
246
+ else:
247
+ raise ValueError(f"Unsupported tensor ndim: {tensor.ndim}")
248
+
249
+
250
+ # ============================================================================
251
+ # FP8 Dequantization Helpers
252
+ # ============================================================================
253
+
254
+
255
+ def dequantize_fp8_per_tensor(
256
+ fp8_tensor: torch.Tensor,
257
+ scale: torch.Tensor,
258
+ output_dtype: torch.dtype = torch.bfloat16,
259
+ ) -> torch.Tensor:
260
+ """Dequantize FP8 tensor with per-tensor scale back to output_dtype."""
261
+ return (fp8_tensor.float() * scale.float()).to(output_dtype)
262
+
263
+
264
+ def dequantize_fp8_per_channel(
265
+ fp8_tensor: torch.Tensor,
266
+ scale: torch.Tensor,
267
+ channel_dim: int,
268
+ output_dtype: torch.dtype = torch.bfloat16,
269
+ ) -> torch.Tensor:
270
+ """Dequantize FP8 tensor with per-channel scale back to output_dtype.
271
+
272
+ For 3D tensor (num_loras, N, K) with channel_dim=1:
273
+ scale shape is (num_loras, N), broadcast over K.
274
+ """
275
+ expand_scale = scale.float()
276
+ # Add trailing dims for broadcasting
277
+ for _ in range(channel_dim + 1, fp8_tensor.ndim):
278
+ expand_scale = expand_scale.unsqueeze(-1)
279
+ return (fp8_tensor.float() * expand_scale).to(output_dtype)
280
+
281
+
282
+ def dequantize_fp8_per_token(
283
+ fp8_tensor: torch.Tensor,
284
+ scale: torch.Tensor,
285
+ output_dtype: torch.dtype = torch.bfloat16,
286
+ ) -> torch.Tensor:
287
+ """Dequantize FP8 2D tensor with per-token scale back to output_dtype.
288
+
289
+ fp8_tensor: (num_tokens, hidden_size), scale: (num_tokens, 1)
290
+ """
291
+ return (fp8_tensor.float() * scale.float()).to(output_dtype)
292
+
293
+
294
+ def dequantize_fp8_blockwise(
295
+ fp8_tensor: torch.Tensor,
296
+ scale: torch.Tensor,
297
+ group_n: int,
298
+ group_k: int,
299
+ output_dtype: torch.dtype = torch.bfloat16,
300
+ ) -> torch.Tensor:
301
+ """Dequantize FP8 tensor with block-wise scale back to output_dtype."""
302
+ if fp8_tensor.ndim == 2:
303
+ M, K = fp8_tensor.shape
304
+ out = torch.zeros(M, K, dtype=output_dtype, device=fp8_tensor.device)
305
+ n_blocks_k = math.ceil(K / group_k)
306
+ for m in range(M):
307
+ for bk in range(n_blocks_k):
308
+ k_start = bk * group_k
309
+ k_end = min(k_start + group_k, K)
310
+ out[m, k_start:k_end] = (
311
+ fp8_tensor[m, k_start:k_end].float() * scale[m, bk].float()
312
+ ).to(output_dtype)
313
+ return out
314
+ elif fp8_tensor.ndim == 3:
315
+ L, N, K = fp8_tensor.shape
316
+ out = torch.zeros(L, N, K, dtype=output_dtype, device=fp8_tensor.device)
317
+ n_blocks_n = math.ceil(N / group_n)
318
+ n_blocks_k = math.ceil(K / group_k)
319
+ for l_idx in range(L):
320
+ for bn in range(n_blocks_n):
321
+ for bk in range(n_blocks_k):
322
+ n_start = bn * group_n
323
+ n_end = min(n_start + group_n, N)
324
+ k_start = bk * group_k
325
+ k_end = min(k_start + group_k, K)
326
+ out[l_idx, n_start:n_end, k_start:k_end] = (
327
+ fp8_tensor[l_idx, n_start:n_end, k_start:k_end].float()
328
+ * scale[l_idx, bn, bk].float()
329
+ ).to(output_dtype)
330
+ return out
331
+ else:
332
+ raise ValueError(f"Unsupported tensor ndim: {fp8_tensor.ndim}")
333
+
334
+
335
+ # ============================================================================
336
+ # FP8 Data Generation
337
+ # ============================================================================
338
+
339
+
340
+ def generate_fp8_shrink_data(
341
+ batches: int,
342
+ hidden_size: int,
343
+ num_loras: int,
344
+ rank: int,
345
+ seq_length: int,
346
+ nslices: int,
347
+ dtype: torch.dtype,
348
+ device: str,
349
+ quant_mode: str, # "per_tensor", "per_channel", "blockwise"
350
+ group_k: int = 128,
351
+ group_n: int = 128,
352
+ ):
353
+ """Generate test data for FP8 shrink kernel.
354
+
355
+ Shrink: output = input @ lora_a^T * scaling
356
+ input: (num_tokens, hidden_size) -> quantized to FP8
357
+ lora_a: (num_loras, rank, hidden_size) -> quantized to FP8
358
+
359
+ Returns bf16 reference tensors, FP8 quantized tensors with scales,
360
+ and dequantized bf16 tensors for accurate reference computation.
361
+ """
362
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1, (batches,)).to(device)
363
+ b_seq_start_loc = torch.cumsum(
364
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
365
+ dim=0,
366
+ ).to(device)
367
+ total_tokens = seq_len_tensor.sum().item()
368
+
369
+ # Generate bf16 reference data
370
+ inputs_bf16 = torch.randn(total_tokens, hidden_size, dtype=dtype, device=device)
371
+
372
+ lora_a_weights_bf16 = []
373
+ for _ in range(nslices):
374
+ lora_a_weights_bf16.append(
375
+ torch.randn(num_loras, rank, hidden_size, dtype=dtype, device=device)
376
+ )
377
+
378
+ # Quantize inputs to FP8 and dequantize back for reference
379
+ if quant_mode == "blockwise":
380
+ inputs_fp8, a_scale = quantize_to_fp8_blockwise(
381
+ inputs_bf16, group_n=1, group_k=group_k
382
+ )
383
+ inputs_dequant = dequantize_fp8_blockwise(
384
+ inputs_fp8,
385
+ a_scale,
386
+ group_n=1,
387
+ group_k=group_k,
388
+ output_dtype=dtype,
389
+ )
390
+ elif quant_mode == "per_tensor":
391
+ # Per-tensor: kernel loads a single scalar from a_scale_ptr
392
+ inputs_fp8, a_scale = quantize_to_fp8_per_tensor(inputs_bf16)
393
+ inputs_dequant = dequantize_fp8_per_tensor(
394
+ inputs_fp8,
395
+ a_scale,
396
+ output_dtype=dtype,
397
+ )
398
+ else:
399
+ # per_channel: kernel loads per-token a_scale via ram indexing
400
+ inputs_fp8, a_scale = quantize_to_fp8_per_token(inputs_bf16)
401
+ inputs_dequant = dequantize_fp8_per_token(
402
+ inputs_fp8,
403
+ a_scale,
404
+ output_dtype=dtype,
405
+ )
406
+
407
+ # Quantize lora_a weights to FP8 and dequantize back for reference
408
+ b_scales = []
409
+ lora_a_weights_fp8 = []
410
+ lora_a_weights_dequant = []
411
+ for w in lora_a_weights_bf16:
412
+ if quant_mode == "per_tensor":
413
+ w_fp8, w_scale = quantize_to_fp8_per_tensor(w)
414
+ w_dequant = dequantize_fp8_per_tensor(w_fp8, w_scale, output_dtype=dtype)
415
+ # Scale shape: (1,) -> need (num_loras,) for the kernel
416
+ w_scale = w_scale.expand(num_loras).contiguous()
417
+ lora_a_weights_fp8.append(w_fp8)
418
+ b_scales.append(w_scale)
419
+ lora_a_weights_dequant.append(w_dequant)
420
+ elif quant_mode == "per_channel":
421
+ # Per-channel along rank dim: scale shape (num_loras, rank)
422
+ w_fp8, w_scale = quantize_to_fp8_per_channel(w, channel_dim=1)
423
+ w_dequant = dequantize_fp8_per_channel(
424
+ w_fp8,
425
+ w_scale,
426
+ channel_dim=1,
427
+ output_dtype=dtype,
428
+ )
429
+ lora_a_weights_fp8.append(w_fp8)
430
+ b_scales.append(w_scale)
431
+ lora_a_weights_dequant.append(w_dequant)
432
+ elif quant_mode == "blockwise":
433
+ w_fp8, w_scale = quantize_to_fp8_blockwise(
434
+ w, group_n=group_n, group_k=group_k
435
+ )
436
+ w_dequant = dequantize_fp8_blockwise(
437
+ w_fp8,
438
+ w_scale,
439
+ group_n=group_n,
440
+ group_k=group_k,
441
+ output_dtype=dtype,
442
+ )
443
+ lora_a_weights_fp8.append(w_fp8)
444
+ b_scales.append(w_scale)
445
+ lora_a_weights_dequant.append(w_dequant)
446
+
447
+ # Output tensor (float32 for shrink)
448
+ out_tensor = torch.zeros(
449
+ nslices, total_tokens, rank, dtype=torch.float32, device=device
450
+ )
451
+ ref_out_tensor = out_tensor.clone()
452
+
453
+ # Token-to-lora mapping
454
+ lora_indices_tensor = torch.randint(0, max(num_loras - 1, 1), (batches,)).to(device)
455
+ token_lora_mapping = torch.zeros(total_tokens, dtype=torch.long, device=device)
456
+ current_offset = 0
457
+ for b_id in range(batches):
458
+ lora_index = lora_indices_tensor[b_id]
459
+ sl = seq_len_tensor[b_id].item()
460
+ token_lora_mapping[current_offset : current_offset + sl] = lora_index
461
+ current_offset += sl
462
+
463
+ return {
464
+ "inputs_bf16": inputs_bf16,
465
+ "inputs_fp8": inputs_fp8,
466
+ "inputs_dequant": inputs_dequant,
467
+ "lora_a_bf16": lora_a_weights_bf16,
468
+ "lora_a_fp8": lora_a_weights_fp8,
469
+ "lora_a_dequant": lora_a_weights_dequant,
470
+ "a_scale": a_scale,
471
+ "b_scales": b_scales,
472
+ "out_tensor": out_tensor,
473
+ "ref_out_tensor": ref_out_tensor,
474
+ "token_lora_mapping": token_lora_mapping,
475
+ "seq_len_tensor": seq_len_tensor,
476
+ "b_seq_start_loc": b_seq_start_loc,
477
+ "lora_indices_tensor": lora_indices_tensor,
478
+ "total_tokens": total_tokens,
479
+ }
480
+
481
+
482
+ def generate_fp8_expand_data(
483
+ batches: int,
484
+ hidden_size: int,
485
+ num_loras: int,
486
+ rank: int,
487
+ seq_length: int,
488
+ nslices: int,
489
+ dtype: torch.dtype,
490
+ device: str,
491
+ quant_mode: str, # "per_tensor", "per_channel", "blockwise"
492
+ group_k: int = 128,
493
+ group_n: int = 128,
494
+ ):
495
+ """Generate test data for FP8 expand kernel (w8a8).
496
+
497
+ Expand: output += input @ lora_b^T
498
+ input: (nslices, num_tokens, rank) -> quantized to FP8 (activations)
499
+ lora_b: (num_loras, hidden_size, rank) -> quantized to FP8 (weights)
500
+
501
+ In w8a8 mode, both activations and weights are FP8.
502
+ Returns bf16 reference tensors, FP8 quantized tensors with scales,
503
+ and dequantized bf16 tensors for accurate reference computation.
504
+ """
505
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1, (batches,)).to(device)
506
+ b_seq_start_loc = torch.cumsum(
507
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
508
+ dim=0,
509
+ ).to(device)
510
+ total_tokens = seq_len_tensor.sum().item()
511
+
512
+ # Generate bf16 input (shrink output) and quantize to FP8
513
+ inputs_bf16 = torch.randn(nslices, total_tokens, rank, dtype=dtype, device=device)
514
+
515
+ # Quantize input to FP8 and dequantize back for reference
516
+ inputs_2d_all = inputs_bf16.reshape(-1, rank)
517
+ if quant_mode == "blockwise":
518
+ # For blockwise, the kernel indexes a_scale by token id (0..total_tokens-1)
519
+ # shared across slices. Compute shared scale across slices, then quantize.
520
+ # First compute per-token-per-block scale across all slices
521
+ n_blocks_k = math.ceil(rank / group_k)
522
+ a_scale = torch.zeros(
523
+ total_tokens, n_blocks_k, dtype=torch.float32, device=device
524
+ )
525
+ for m in range(total_tokens):
526
+ for bk in range(n_blocks_k):
527
+ k_start = bk * group_k
528
+ k_end = min(k_start + group_k, rank)
529
+ # Max across all slices for this token and block
530
+ block_amax = torch.tensor(0.0, device=device)
531
+ for s in range(nslices):
532
+ block = inputs_bf16[s, m, k_start:k_end].float()
533
+ block_amax = torch.max(
534
+ block_amax, block.abs().max().clamp(min=1e-12)
535
+ )
536
+ a_scale[m, bk] = (block_amax / FP8_MAX).to(torch.float32)
537
+
538
+ # Quantize all slices with the shared scale
539
+ inputs_fp8_list = []
540
+ inputs_dequant_list = []
541
+ for s in range(nslices):
542
+ slice_2d = inputs_bf16[s] # (total_tokens, rank)
543
+ fp8_slice = torch.zeros_like(slice_2d, dtype=FP8_DTYPE)
544
+ dequant_slice = torch.zeros_like(slice_2d)
545
+ for m in range(total_tokens):
546
+ for bk in range(n_blocks_k):
547
+ k_start = bk * group_k
548
+ k_end = min(k_start + group_k, rank)
549
+ block = slice_2d[m, k_start:k_end].float()
550
+ s_val = a_scale[m, bk]
551
+ fp8_slice[m, k_start:k_end] = (
552
+ (block / s_val).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
553
+ )
554
+ dequant_slice[m, k_start:k_end] = (
555
+ fp8_slice[m, k_start:k_end].float() * s_val.float()
556
+ ).to(dtype)
557
+ inputs_fp8_list.append(fp8_slice)
558
+ inputs_dequant_list.append(dequant_slice)
559
+ inputs_fp8 = torch.stack(inputs_fp8_list, dim=0)
560
+ inputs_dequant = torch.stack(inputs_dequant_list, dim=0)
561
+ elif quant_mode == "per_tensor":
562
+ # Per-tensor: kernel loads a single scalar from a_scale_ptr
563
+ inputs_fp8_2d, a_scale = quantize_to_fp8_per_tensor(inputs_2d_all)
564
+ inputs_dequant_2d = dequantize_fp8_per_tensor(
565
+ inputs_fp8_2d,
566
+ a_scale,
567
+ output_dtype=dtype,
568
+ )
569
+ inputs_fp8 = inputs_fp8_2d.reshape(nslices, total_tokens, rank)
570
+ inputs_dequant = inputs_dequant_2d.reshape(nslices, total_tokens, rank)
571
+ else:
572
+ # per_channel: kernel loads per-token a_scale via ram indexing.
573
+ # The kernel uses the same a_scale for all slices (indexed by token
574
+ # id 0..total_tokens-1), so we compute a shared per-token scale
575
+ # across all slices, then quantize each slice with that shared scale.
576
+ per_slice_views = [inputs_bf16[s] for s in range(nslices)]
577
+ # (nslices, total_tokens, rank) -> max across slices per token
578
+ stacked = torch.stack(per_slice_views, dim=0) # (nslices, tokens, rank)
579
+ amax = stacked.abs().float().amax(dim=(0, 2), keepdim=False).clamp(min=1e-12)
580
+ # amax shape: (total_tokens,)
581
+ a_scale = (amax / FP8_MAX).to(torch.float32).unsqueeze(1) # (tokens, 1)
582
+ # Quantize all slices with the shared scale
583
+ inputs_fp8_2d = (
584
+ (inputs_2d_all.float() / a_scale.repeat(nslices, 1))
585
+ .clamp(FP8_MIN, FP8_MAX)
586
+ .to(FP8_DTYPE)
587
+ )
588
+ inputs_dequant_2d = (
589
+ inputs_fp8_2d.float() * a_scale.repeat(nslices, 1).float()
590
+ ).to(dtype)
591
+ inputs_fp8 = inputs_fp8_2d.reshape(nslices, total_tokens, rank)
592
+ inputs_dequant = inputs_dequant_2d.reshape(nslices, total_tokens, rank)
593
+
594
+ # Generate bf16 LoRA B weights
595
+ lora_b_weights_bf16 = []
596
+ for _ in range(nslices):
597
+ lora_b_weights_bf16.append(
598
+ torch.randn(num_loras, hidden_size, rank, dtype=dtype, device=device)
599
+ )
600
+
601
+ # Quantize LoRA B weights to FP8 and dequantize back for reference
602
+ b_scales = []
603
+ lora_b_weights_fp8 = []
604
+ lora_b_weights_dequant = []
605
+ for w in lora_b_weights_bf16:
606
+ if quant_mode == "per_tensor":
607
+ w_fp8, w_scale = quantize_to_fp8_per_tensor(w)
608
+ w_dequant = dequantize_fp8_per_tensor(w_fp8, w_scale, output_dtype=dtype)
609
+ w_scale = w_scale.expand(num_loras).contiguous()
610
+ lora_b_weights_fp8.append(w_fp8)
611
+ b_scales.append(w_scale)
612
+ lora_b_weights_dequant.append(w_dequant)
613
+ elif quant_mode == "per_channel":
614
+ # Per-channel along hidden_size dim: scale (num_loras, hidden_size)
615
+ w_fp8, w_scale = quantize_to_fp8_per_channel(w, channel_dim=1)
616
+ w_dequant = dequantize_fp8_per_channel(
617
+ w_fp8,
618
+ w_scale,
619
+ channel_dim=1,
620
+ output_dtype=dtype,
621
+ )
622
+ lora_b_weights_fp8.append(w_fp8)
623
+ b_scales.append(w_scale)
624
+ lora_b_weights_dequant.append(w_dequant)
625
+ elif quant_mode == "blockwise":
626
+ w_fp8, w_scale = quantize_to_fp8_blockwise(
627
+ w, group_n=group_n, group_k=group_k
628
+ )
629
+ w_dequant = dequantize_fp8_blockwise(
630
+ w_fp8,
631
+ w_scale,
632
+ group_n=group_n,
633
+ group_k=group_k,
634
+ output_dtype=dtype,
635
+ )
636
+ lora_b_weights_fp8.append(w_fp8)
637
+ b_scales.append(w_scale)
638
+ lora_b_weights_dequant.append(w_dequant)
639
+
640
+ # Output tensor (initialized randomly for add_inputs)
641
+ out_tensor = torch.randn(
642
+ total_tokens, hidden_size * nslices, dtype=dtype, device=device
643
+ )
644
+ ref_out_tensor = out_tensor.clone()
645
+
646
+ # Token-to-lora mapping
647
+ lora_indices_tensor = torch.randint(0, max(num_loras - 1, 1), (batches,)).to(device)
648
+ token_lora_mapping = torch.zeros(total_tokens, dtype=torch.long, device=device)
649
+ current_offset = 0
650
+ for b_id in range(batches):
651
+ lora_index = lora_indices_tensor[b_id]
652
+ sl = seq_len_tensor[b_id].item()
653
+ token_lora_mapping[current_offset : current_offset + sl] = lora_index
654
+ current_offset += sl
655
+
656
+ return {
657
+ "inputs_bf16": inputs_bf16,
658
+ "inputs_fp8": inputs_fp8,
659
+ "inputs_dequant": inputs_dequant,
660
+ "a_scale": a_scale,
661
+ "lora_b_bf16": lora_b_weights_bf16,
662
+ "lora_b_fp8": lora_b_weights_fp8,
663
+ "lora_b_dequant": lora_b_weights_dequant,
664
+ "b_scales": b_scales,
665
+ "out_tensor": out_tensor,
666
+ "ref_out_tensor": ref_out_tensor,
667
+ "token_lora_mapping": token_lora_mapping,
668
+ "seq_len_tensor": seq_len_tensor,
669
+ "b_seq_start_loc": b_seq_start_loc,
670
+ "lora_indices_tensor": lora_indices_tensor,
671
+ "total_tokens": total_tokens,
672
+ }
673
+
674
+
675
+ # ============================================================================
676
+ # FP8 Shrink Kernel Check
677
+ # ============================================================================
678
+
679
+
680
+ def check_lora_shrink_fp8_kernel(
681
+ batches: int,
682
+ num_loras: int,
683
+ rank: int,
684
+ hidden_size: int,
685
+ nslices: int,
686
+ dtype: torch.dtype,
687
+ device: str,
688
+ seq_length: int,
689
+ scaling: float,
690
+ quant_mode: str,
691
+ group_k: int = 128,
692
+ group_n: int = 128,
693
+ ):
694
+ """Test FP8 shrink kernel against dequantized bf16 reference.
695
+
696
+ Instead of comparing FP8 kernel output against the original bf16 reference
697
+ (which conflates quantization error with kernel error), we:
698
+ 1. Quantize bf16 inputs/weights to FP8
699
+ 2. Dequantize them back to bf16
700
+ 3. Run the bf16 reference (sgmv_shrink) with the dequantized values
701
+ 4. Compare FP8 kernel output against this dequantized reference
702
+
703
+ This isolates kernel correctness from quantization precision loss,
704
+ allowing much tighter tolerances.
705
+ """
706
+ data = generate_fp8_shrink_data(
707
+ batches,
708
+ hidden_size,
709
+ num_loras,
710
+ rank,
711
+ seq_length,
712
+ nslices,
713
+ dtype,
714
+ device,
715
+ quant_mode,
716
+ group_k,
717
+ group_n,
718
+ )
719
+
720
+ total_tokens = data["total_tokens"]
721
+
722
+ # Setup LoRA kernel metadata
723
+ lora_meta = LoRAKernelMeta.make(
724
+ max_loras=num_loras, max_num_tokens=total_tokens, device=device
725
+ )
726
+ lora_meta.prepare_tensors(data["token_lora_mapping"])
727
+
728
+ out_tensor = data["out_tensor"]
729
+
730
+ # Determine quantization params for the kernel
731
+ per_channel = quant_mode == "per_channel"
732
+ gk = group_k if quant_mode == "blockwise" else 0
733
+ gn = group_n if quant_mode == "blockwise" else 0
734
+
735
+ with _dict_lock:
736
+ _LORA_A_PTR_DICT.clear()
737
+ _SHRINK_LORA_SCALE_PTR_DICT.clear()
738
+ triton_ops.lora_shrink_fp8(
739
+ data["inputs_fp8"],
740
+ data["lora_a_fp8"],
741
+ out_tensor,
742
+ *lora_meta.meta_args(token_nums=total_tokens, specialize_active_lora=False),
743
+ scaling,
744
+ data["b_scales"],
745
+ a_scale=data["a_scale"],
746
+ group_k=gk,
747
+ group_n=gn,
748
+ use_fp8_w8a8=True,
749
+ per_channel_quant=per_channel,
750
+ )
751
+
752
+ # Compute reference using dequantized (round-tripped) tensors.
753
+ # This means the reference sees the same quantization error as the kernel,
754
+ # so any difference is purely kernel error.
755
+ ref_out_tensor = data["ref_out_tensor"]
756
+ max_seq_length = data["seq_len_tensor"].max().item()
757
+ sgmv_shrink_for_nslices(
758
+ nslices,
759
+ data["inputs_dequant"],
760
+ data["lora_a_dequant"],
761
+ ref_out_tensor,
762
+ data["b_seq_start_loc"],
763
+ data["seq_len_tensor"],
764
+ data["lora_indices_tensor"],
765
+ batches,
766
+ max_seq_length,
767
+ total_tokens,
768
+ scaling,
769
+ )
770
+
771
+ # With dequantized reference, we can use much tighter tolerances
772
+ # since we're only measuring kernel error, not quantization error.
773
+ # Blockwise accumulation order differs from the bf16 reference, so
774
+ # allow a slightly larger margin for sporadic rounding outliers.
775
+ rtol, atol = 0.1, 0.25
776
+ torch.testing.assert_close(
777
+ out_tensor.to(dtype), ref_out_tensor.to(dtype), rtol=rtol, atol=atol
778
+ )
779
+
780
+
781
+ # ============================================================================
782
+ # FP8 Expand Kernel Check
783
+ # ============================================================================
784
+
785
+
786
+ def check_lora_expand_fp8_kernel(
787
+ batches: int,
788
+ num_loras: int,
789
+ rank: int,
790
+ hidden_size: int,
791
+ nslices: int,
792
+ dtype: torch.dtype,
793
+ device: str,
794
+ seq_length: int,
795
+ add_inputs: bool,
796
+ quant_mode: str,
797
+ group_k: int = 128,
798
+ group_n: int = 128,
799
+ ):
800
+ """Test FP8 expand kernel (w8a8) against dequantized bf16 reference.
801
+
802
+ Instead of comparing FP8 kernel output against the original bf16 reference
803
+ (which conflates quantization error with kernel error), we:
804
+ 1. Quantize bf16 inputs/weights to FP8
805
+ 2. Dequantize them back to bf16
806
+ 3. Run the bf16 reference (sgmv_expand) with the dequantized values
807
+ 4. Compare FP8 kernel output against this dequantized reference
808
+
809
+ This isolates kernel correctness from quantization precision loss,
810
+ allowing much tighter tolerances.
811
+ """
812
+ data = generate_fp8_expand_data(
813
+ batches,
814
+ hidden_size,
815
+ num_loras,
816
+ rank,
817
+ seq_length,
818
+ nslices,
819
+ dtype,
820
+ device,
821
+ quant_mode,
822
+ group_k,
823
+ group_n,
824
+ )
825
+
826
+ total_tokens = data["total_tokens"]
827
+
828
+ # Setup LoRA kernel metadata
829
+ lora_meta = LoRAKernelMeta.make(
830
+ max_loras=num_loras, max_num_tokens=total_tokens, device=device
831
+ )
832
+ lora_meta.prepare_tensors(data["token_lora_mapping"])
833
+
834
+ out_tensor = data["out_tensor"]
835
+
836
+ # Determine quantization params for the kernel
837
+ per_channel = quant_mode == "per_channel"
838
+ gk = group_k if quant_mode == "blockwise" else 0
839
+ gn = group_n if quant_mode == "blockwise" else 0
840
+
841
+ with _dict_lock:
842
+ _LORA_B_PTR_DICT.clear()
843
+ _EXPAND_LORA_SCALE_PTR_DICT.clear()
844
+ triton_ops.lora_expand_fp8(
845
+ data["inputs_fp8"],
846
+ data["lora_b_fp8"],
847
+ out_tensor,
848
+ *lora_meta.meta_args(token_nums=total_tokens, specialize_active_lora=False),
849
+ data["b_scales"],
850
+ a_scale=data["a_scale"],
851
+ offset_start=0,
852
+ add_inputs=add_inputs,
853
+ group_k=gk,
854
+ group_n=gn,
855
+ use_fp8_w8a8=True,
856
+ per_channel_quant=per_channel,
857
+ )
858
+
859
+ # Compute reference using dequantized (round-tripped) tensors.
860
+ ref_out_tensor = data["ref_out_tensor"]
861
+ max_seq_length = data["seq_len_tensor"].max().item()
862
+ sgmv_expand_for_nslices(
863
+ nslices,
864
+ hidden_size,
865
+ data["inputs_dequant"],
866
+ data["lora_b_dequant"],
867
+ ref_out_tensor,
868
+ data["b_seq_start_loc"],
869
+ data["seq_len_tensor"],
870
+ data["lora_indices_tensor"],
871
+ batches,
872
+ max_seq_length,
873
+ total_tokens,
874
+ add_inputs=add_inputs,
875
+ )
876
+
877
+ # With dequantized reference, we can use much tighter tolerances
878
+ # since we're only measuring kernel error, not quantization error.
879
+ rtol, atol = 0.1, 0.15
880
+ torch.testing.assert_close(out_tensor, ref_out_tensor, rtol=rtol, atol=atol)
881
+
882
+
883
+ # ============================================================================
884
+ # FP8 Test Parameters
885
+ # ============================================================================
886
+
887
+ fp8_test_params = {
888
+ "hidden_sizes": [512, 1024, 2048],
889
+ "batches": [1, 4, 16],
890
+ "num_loras": [1, 4, 8],
891
+ "max_ranks": [8, 16, 32, 64],
892
+ }
893
+
894
+
895
+ # ============================================================================
896
+ # FP8 Shrink Tests
897
+ # ============================================================================
898
+
899
+
900
+ @pytest.mark.parametrize("batches", fp8_test_params["batches"])
901
+ @pytest.mark.parametrize("num_loras", fp8_test_params["num_loras"])
902
+ @pytest.mark.parametrize("rank", fp8_test_params["max_ranks"])
903
+ @pytest.mark.parametrize("hidden_size", fp8_test_params["hidden_sizes"])
904
+ @pytest.mark.parametrize("nslices", [1, 2, 3])
905
+ @pytest.mark.parametrize("dtype", [torch.bfloat16])
906
+ @pytest.mark.parametrize("device", DEVICES)
907
+ @pytest.mark.parametrize("seed", SEED)
908
+ @pytest.mark.parametrize("quant_mode", ["per_tensor", "per_channel", "blockwise"])
909
+ def test_lora_shrink_fp8(
910
+ batches: int,
911
+ num_loras: int,
912
+ rank: int,
913
+ hidden_size: int,
914
+ nslices: int,
915
+ dtype: torch.dtype,
916
+ device: str,
917
+ seed: int,
918
+ quant_mode: str,
919
+ ):
920
+ """Test FP8 shrink kernel with per-tensor, per-channel, and block-wise
921
+ quantization, comparing against the bf16 baseline."""
922
+ torch.set_default_device(device)
923
+ set_random_seed(seed)
924
+
925
+ # For blockwise, group sizes must divide evenly or be handled by the kernel
926
+ group_k = 128
927
+ group_n = 128
928
+
929
+ # Adjust group sizes if they're larger than the dimensions
930
+ if quant_mode == "blockwise":
931
+ group_k = min(group_k, hidden_size)
932
+ group_n = min(group_n, rank)
933
+
934
+ check_lora_shrink_fp8_kernel(
935
+ batches=batches,
936
+ num_loras=num_loras,
937
+ rank=rank,
938
+ hidden_size=hidden_size,
939
+ nslices=nslices,
940
+ dtype=dtype,
941
+ device=device,
942
+ seq_length=128,
943
+ scaling=0.5,
944
+ quant_mode=quant_mode,
945
+ group_k=group_k,
946
+ group_n=group_n,
947
+ )
948
+
949
+
950
+ # ============================================================================
951
+ # FP8 Expand Tests
952
+ # ============================================================================
953
+
954
+
955
+ @pytest.mark.parametrize("batches", fp8_test_params["batches"])
956
+ @pytest.mark.parametrize("num_loras", fp8_test_params["num_loras"])
957
+ @pytest.mark.parametrize("rank", fp8_test_params["max_ranks"])
958
+ @pytest.mark.parametrize("hidden_size", fp8_test_params["hidden_sizes"])
959
+ @pytest.mark.parametrize("nslices", [1, 2, 3])
960
+ @pytest.mark.parametrize("dtype", [torch.bfloat16])
961
+ @pytest.mark.parametrize("device", DEVICES)
962
+ @pytest.mark.parametrize("seed", SEED)
963
+ @pytest.mark.parametrize("quant_mode", ["per_tensor", "per_channel", "blockwise"])
964
+ def test_lora_expand_fp8(
965
+ batches: int,
966
+ num_loras: int,
967
+ rank: int,
968
+ hidden_size: int,
969
+ nslices: int,
970
+ dtype: torch.dtype,
971
+ device: str,
972
+ seed: int,
973
+ quant_mode: str,
974
+ ):
975
+ """Test FP8 expand kernel with per-tensor, per-channel, and block-wise
976
+ quantization, comparing against the bf16 baseline."""
977
+ torch.set_default_device(device)
978
+ set_random_seed(seed)
979
+
980
+ group_k = 128
981
+ group_n = 128
982
+
983
+ # Adjust group sizes if they're larger than the dimensions
984
+ if quant_mode == "blockwise":
985
+ group_k = min(group_k, rank)
986
+ group_n = min(group_n, hidden_size)
987
+
988
+ check_lora_expand_fp8_kernel(
989
+ batches=batches,
990
+ num_loras=num_loras,
991
+ rank=rank,
992
+ hidden_size=hidden_size,
993
+ nslices=nslices,
994
+ dtype=dtype,
995
+ device=device,
996
+ seq_length=128,
997
+ add_inputs=True,
998
+ quant_mode=quant_mode,
999
+ group_k=group_k,
1000
+ group_n=group_n,
1001
+ )
asian_ai/performance/vLLM/tests/lora/test_punica_xpu_ops.py ADDED
@@ -0,0 +1,298 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import pytest
5
+ import torch
6
+
7
+ from tests.lora.utils import (
8
+ PunicaTensors,
9
+ assert_close,
10
+ generate_data,
11
+ generate_data_for_expand_nslices,
12
+ )
13
+ from vllm.lora.ops.xpu_ops import bgmv_expand, bgmv_expand_slice, bgmv_shrink
14
+ from vllm.platforms import current_platform
15
+
16
+
17
+ def torch_bgmv_expand(
18
+ inputs: torch.Tensor,
19
+ lora_b_weights: torch.Tensor,
20
+ output_tensor: torch.Tensor,
21
+ lora_indices_tensor: torch.Tensor,
22
+ add_inputs: bool = True,
23
+ ):
24
+ selected_loras = lora_b_weights[lora_indices_tensor].to(dtype=output_tensor.dtype)
25
+ if len(selected_loras.shape) == 4:
26
+ selected_loras = selected_loras.squeeze(dim=1)
27
+ inputs = inputs.to(dtype=output_tensor.dtype)
28
+ outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras)
29
+
30
+ limit = output_tensor.shape[0]
31
+ if outputs.shape[0] == 1 and output_tensor.shape[0] != 1:
32
+ limit = 1
33
+
34
+ # LoRA adapter and model may add different amounts of padding to output
35
+ common_len = min(outputs.shape[1], output_tensor.shape[1])
36
+
37
+ if add_inputs:
38
+ output_tensor[:, :common_len] += outputs[:limit, :common_len]
39
+ else:
40
+ output_tensor[:, :common_len] = outputs[:limit, :common_len]
41
+
42
+
43
+ def torch_bgmv_shrink(
44
+ inputs: torch.Tensor,
45
+ lora_b_weights: torch.Tensor,
46
+ output_tensor: torch.Tensor,
47
+ lora_indices_tensor: torch.Tensor,
48
+ scaling: float = 1.0,
49
+ ):
50
+ selected_loras = lora_b_weights[lora_indices_tensor].to(dtype=output_tensor.dtype)
51
+ if len(selected_loras.shape) == 4:
52
+ selected_loras = selected_loras.squeeze(dim=1)
53
+ inputs = inputs.to(dtype=output_tensor.dtype)
54
+ outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras)
55
+
56
+ output_tensor[:, : outputs.shape[1]] = scaling * outputs[:]
57
+
58
+
59
+ def torch_bgmv_expand_slice(
60
+ inputs: torch.Tensor,
61
+ lora_b_weights: torch.Tensor,
62
+ output_tensor: torch.Tensor,
63
+ lora_indices_tensor: torch.Tensor,
64
+ slice_offset: int,
65
+ slice_size: int,
66
+ add_inputs: bool = True,
67
+ ):
68
+ selected_loras = lora_b_weights[lora_indices_tensor].to(dtype=output_tensor.dtype)
69
+ inputs = inputs.to(dtype=output_tensor.dtype)
70
+ if len(selected_loras.shape) == 4:
71
+ selected_loras = selected_loras.squeeze(dim=1)
72
+ outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras)
73
+
74
+ if add_inputs:
75
+ output_tensor[:, slice_offset : slice_offset + slice_size] += outputs[:]
76
+ else:
77
+ output_tensor[:, slice_offset : slice_offset + slice_size] = outputs[:]
78
+
79
+
80
+ def check_bgmv_shrink(
81
+ batches: int,
82
+ num_loras: int,
83
+ rank: int,
84
+ hidden_size: int,
85
+ dtype: torch.dtype,
86
+ device: str,
87
+ scaling: float,
88
+ ):
89
+ """
90
+ Compare vllm.bgmv_shrink against a reference implementation.
91
+ """
92
+ seq_length = 1
93
+ data: PunicaTensors = generate_data(
94
+ batches,
95
+ hidden_size,
96
+ num_loras,
97
+ rank,
98
+ seq_length,
99
+ dtype,
100
+ "shrink",
101
+ device,
102
+ )
103
+
104
+ bgmv_shrink(
105
+ data.inputs_tensor,
106
+ data.lora_weights,
107
+ data.our_out_tensor,
108
+ data.token_lora_mapping,
109
+ scaling,
110
+ )
111
+
112
+ torch_bgmv_shrink(
113
+ data.inputs_tensor,
114
+ data.lora_weights,
115
+ data.ref_out_tensor,
116
+ data.token_lora_mapping,
117
+ scaling,
118
+ )
119
+
120
+ data.ref_out_tensor = data.ref_out_tensor.to(torch.float32)
121
+ assert_close(data.our_out_tensor, data.ref_out_tensor)
122
+
123
+
124
+ def check_bgmv_expand(
125
+ batches: int,
126
+ num_loras: int,
127
+ rank: int,
128
+ hidden_size: int,
129
+ dtype: torch.dtype,
130
+ device: str,
131
+ add_inputs: bool,
132
+ ):
133
+ """
134
+ Compare vllm.bgmv_expand against a reference implementation.
135
+ """
136
+ seq_length = 1
137
+ data: PunicaTensors = generate_data(
138
+ batches,
139
+ hidden_size,
140
+ num_loras,
141
+ rank,
142
+ seq_length,
143
+ dtype,
144
+ "expand",
145
+ device,
146
+ )
147
+
148
+ bgmv_expand(
149
+ data.inputs_tensor,
150
+ data.lora_weights,
151
+ data.our_out_tensor,
152
+ data.token_lora_mapping,
153
+ add_inputs=add_inputs,
154
+ )
155
+ torch_bgmv_expand(
156
+ data.inputs_tensor,
157
+ data.lora_weights,
158
+ data.ref_out_tensor,
159
+ data.token_lora_mapping,
160
+ add_inputs=add_inputs,
161
+ )
162
+ assert_close(data.ref_out_tensor, data.our_out_tensor)
163
+
164
+
165
+ def check_bgmv_expand_slice(
166
+ batches: int,
167
+ num_loras: int,
168
+ rank: int,
169
+ hidden_size: int,
170
+ nslices: int,
171
+ dtype: torch.dtype,
172
+ device: str,
173
+ add_inputs: bool,
174
+ ):
175
+ """
176
+ Compare vllm.bgmv_expand_slice against a reference implementation.
177
+ """
178
+ seq_length = 1
179
+ data: PunicaTensors = generate_data_for_expand_nslices(
180
+ batches,
181
+ hidden_size,
182
+ num_loras,
183
+ rank,
184
+ seq_length,
185
+ dtype,
186
+ nslices,
187
+ device,
188
+ )
189
+
190
+ slice_offset = 0
191
+ for index in range(nslices):
192
+ bgmv_expand_slice(
193
+ data.inputs_tensor,
194
+ data.lora_weights[index],
195
+ data.our_out_tensor,
196
+ data.token_lora_mapping,
197
+ slice_offset,
198
+ slice_size=hidden_size,
199
+ add_inputs=add_inputs,
200
+ )
201
+ torch_bgmv_expand_slice(
202
+ data.inputs_tensor,
203
+ data.lora_weights[index],
204
+ data.ref_out_tensor,
205
+ data.token_lora_mapping,
206
+ slice_offset,
207
+ slice_size=hidden_size,
208
+ add_inputs=add_inputs,
209
+ )
210
+
211
+ slice_offset += hidden_size
212
+ assert_close(data.ref_out_tensor, data.our_out_tensor)
213
+
214
+
215
+ # General tests params that tests for variations in all dimensions
216
+ # except hidden_size.
217
+ test_params = {
218
+ "hidden_sizes": [2049],
219
+ "batches": [4],
220
+ "num_loras": [4],
221
+ "max_ranks": [32],
222
+ }
223
+
224
+ DTYPES = [torch.float16, torch.bfloat16]
225
+ DEVICES = [f"xpu:{0}"]
226
+ SEED = [0]
227
+
228
+
229
+ @pytest.mark.parametrize("batches", test_params["batches"])
230
+ @pytest.mark.parametrize("num_loras", test_params["num_loras"])
231
+ @pytest.mark.parametrize("rank", test_params["max_ranks"])
232
+ @pytest.mark.parametrize("hidden_size", test_params["hidden_sizes"])
233
+ @pytest.mark.parametrize("dtype", DTYPES)
234
+ @pytest.mark.parametrize("device", DEVICES)
235
+ @pytest.mark.parametrize("seed", SEED)
236
+ @pytest.mark.parametrize("op_type", ["shrink", "expand"])
237
+ @pytest.mark.skipif(not current_platform.is_xpu(), reason="skip for non xpu platform")
238
+ def test_bgmv(
239
+ batches: int,
240
+ num_loras: int,
241
+ rank: int,
242
+ hidden_size: int,
243
+ dtype: torch.dtype,
244
+ device: str,
245
+ seed: int,
246
+ op_type: str,
247
+ ):
248
+ if op_type == "shrink":
249
+ check_bgmv_shrink(
250
+ batches=batches,
251
+ num_loras=num_loras,
252
+ rank=rank,
253
+ hidden_size=hidden_size,
254
+ dtype=dtype,
255
+ device=device,
256
+ scaling=0.5,
257
+ )
258
+ else:
259
+ check_bgmv_expand(
260
+ batches=batches,
261
+ num_loras=num_loras,
262
+ rank=rank,
263
+ hidden_size=hidden_size,
264
+ dtype=dtype,
265
+ device=device,
266
+ add_inputs=True,
267
+ )
268
+
269
+
270
+ @pytest.mark.parametrize("batches", test_params["batches"])
271
+ @pytest.mark.parametrize("num_loras", test_params["num_loras"])
272
+ @pytest.mark.parametrize("rank", test_params["max_ranks"])
273
+ @pytest.mark.parametrize("hidden_size", test_params["hidden_sizes"])
274
+ @pytest.mark.parametrize("nslices", [2, 3])
275
+ @pytest.mark.parametrize("dtype", DTYPES)
276
+ @pytest.mark.parametrize("device", DEVICES)
277
+ @pytest.mark.parametrize("seed", SEED)
278
+ @pytest.mark.skipif(not current_platform.is_xpu(), reason="skip for non xpu platform")
279
+ def test_bgmv_expand_nslices(
280
+ batches: int,
281
+ num_loras: int,
282
+ rank: int,
283
+ hidden_size: int,
284
+ nslices: int,
285
+ dtype: torch.dtype,
286
+ device: str,
287
+ seed: int,
288
+ ):
289
+ check_bgmv_expand_slice(
290
+ batches=batches,
291
+ num_loras=num_loras,
292
+ rank=rank,
293
+ hidden_size=hidden_size,
294
+ nslices=nslices,
295
+ dtype=dtype,
296
+ device=device,
297
+ add_inputs=True,
298
+ )
asian_ai/performance/vLLM/tests/lora/test_quant_model.py ADDED
@@ -0,0 +1,167 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ # Adapted from
5
+ # https://github.com/fmmoret/vllm/blob/fm-support-lora-on-quantized-models/tests/lora/test_llama.py
6
+ from dataclasses import dataclass
7
+
8
+ import pytest
9
+
10
+ import vllm
11
+ from vllm.distributed import cleanup_dist_env_and_memory
12
+ from vllm.lora.request import LoRARequest
13
+ from vllm.platforms import current_platform
14
+
15
+
16
+ @dataclass
17
+ class ModelWithQuantization:
18
+ model_path: str
19
+ quantization: str
20
+
21
+
22
+ MODELS: list[ModelWithQuantization]
23
+ # AWQ quantization is currently not supported in ROCm.
24
+ if current_platform.is_rocm():
25
+ MODELS = [
26
+ ModelWithQuantization(
27
+ model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", quantization="gptq"
28
+ ),
29
+ ]
30
+ else:
31
+ MODELS = [
32
+ ModelWithQuantization(
33
+ model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", quantization="awq"
34
+ ),
35
+ ModelWithQuantization(
36
+ model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", quantization="gptq"
37
+ ),
38
+ ]
39
+
40
+
41
+ def do_sample(
42
+ llm: vllm.LLM, lora_path: str, lora_id: int, max_tokens: int = 256
43
+ ) -> list[str]:
44
+ raw_prompts = [
45
+ "Give me an orange-ish brown color",
46
+ "Give me a neon pink color",
47
+ ]
48
+
49
+ def format_prompt_tuples(prompt):
50
+ return f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
51
+
52
+ prompts = [format_prompt_tuples(p) for p in raw_prompts]
53
+
54
+ sampling_params = vllm.SamplingParams(
55
+ temperature=0, max_tokens=max_tokens, stop=["<|im_end|>"]
56
+ )
57
+ outputs = llm.generate(
58
+ prompts,
59
+ sampling_params,
60
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
61
+ )
62
+ # Print the outputs.
63
+ generated_texts: list[str] = []
64
+ for output in outputs:
65
+ prompt = output.prompt
66
+ generated_text = output.outputs[0].text
67
+ generated_texts.append(generated_text)
68
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
69
+ return generated_texts
70
+
71
+
72
+ @pytest.mark.parametrize("model", MODELS)
73
+ def test_quant_model_lora(tinyllama_lora_files, model):
74
+ llm = vllm.LLM(
75
+ model=model.model_path,
76
+ enable_lora=True,
77
+ max_num_seqs=16,
78
+ max_loras=4,
79
+ max_model_len=400,
80
+ gpu_memory_utilization=0.2, # avoid OOM
81
+ quantization=model.quantization,
82
+ trust_remote_code=True,
83
+ enable_chunked_prefill=True,
84
+ tokenizer=tinyllama_lora_files,
85
+ )
86
+
87
+ if model.quantization is None:
88
+ expected_lora_output = [
89
+ "#ff8050",
90
+ "#ff8080",
91
+ ]
92
+ elif model.quantization == "awq":
93
+ expected_lora_output = [
94
+ "#f07700: A v",
95
+ "#f00000: A v",
96
+ ]
97
+ elif model.quantization == "gptq":
98
+ expected_lora_output = [
99
+ "#f08800: This is",
100
+ "#f07788 \n#",
101
+ ]
102
+
103
+ def expect_match(output, expected_output):
104
+ # HACK: GPTQ lora outputs are just incredibly unstable.
105
+ # Assert that the outputs changed.
106
+ if model.quantization == "gptq" and expected_output is expected_lora_output:
107
+ for i, o in enumerate(output):
108
+ assert o.startswith("#"), (
109
+ f"Expected example {i} to start with # but got {o}"
110
+ )
111
+ return
112
+ assert output == expected_output
113
+
114
+ max_tokens = 10
115
+
116
+ print("lora adapter created")
117
+ print("lora 1")
118
+ output = do_sample(llm, tinyllama_lora_files, lora_id=1, max_tokens=max_tokens)
119
+ expect_match(output, expected_lora_output)
120
+
121
+ print("lora 2")
122
+ output = do_sample(llm, tinyllama_lora_files, lora_id=2, max_tokens=max_tokens)
123
+ expect_match(output, expected_lora_output)
124
+
125
+ print("removing lora")
126
+
127
+ del llm
128
+ cleanup_dist_env_and_memory()
129
+
130
+
131
+ @pytest.mark.parametrize("model", MODELS)
132
+ def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available, model):
133
+ if num_gpus_available < 2:
134
+ pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
135
+ if model.quantization == "gptq":
136
+ pytest.skip("GPTQ lora outputs are just incredibly unstable")
137
+ llm_tp1 = vllm.LLM(
138
+ model=model.model_path,
139
+ enable_lora=True,
140
+ max_num_seqs=16,
141
+ max_loras=4,
142
+ gpu_memory_utilization=0.2, # avoid OOM
143
+ quantization=model.quantization,
144
+ trust_remote_code=True,
145
+ enable_chunked_prefill=True,
146
+ )
147
+ output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
148
+
149
+ del llm_tp1
150
+ cleanup_dist_env_and_memory()
151
+
152
+ llm_tp2 = vllm.LLM(
153
+ model=model.model_path,
154
+ enable_lora=True,
155
+ max_num_seqs=16,
156
+ max_loras=4,
157
+ tensor_parallel_size=2,
158
+ gpu_memory_utilization=0.2, # avoid OOM
159
+ quantization=model.quantization,
160
+ enable_chunked_prefill=True,
161
+ )
162
+ output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
163
+
164
+ del llm_tp2
165
+ cleanup_dist_env_and_memory()
166
+
167
+ assert output_tp1 == output_tp2
asian_ai/performance/vLLM/tests/lora/test_qwen35_densemodel_lora.py ADDED
@@ -0,0 +1,370 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import pytest
5
+ from transformers import AutoTokenizer
6
+
7
+ import vllm
8
+ import vllm.config
9
+ from vllm.assets.image import ImageAsset
10
+ from vllm.lora.request import LoRARequest
11
+ from vllm.platforms import current_platform
12
+
13
+ from ..utils import create_new_process_for_each_test, multi_gpu_test
14
+
15
+ MODEL_PATH = "Qwen/Qwen3.5-4B"
16
+ TEXT_LORA_ID = 1
17
+ VL_LORA_ID = 2
18
+
19
+ # text-only task
20
+ TEXT_PROMPT_TEMPLATE = """Write a SQL query for the given database.\nSchema:\nTables:\n - stadium(Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average)\n - singer(Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male)\n - concert(concert_ID, concert_Name, Theme, Stadium_ID, Year)\n - singer_in_concert(concert_ID, Singer_ID)\n\nQuestion:\n{query}""" # noqa: E501
21
+
22
+ TEXT_EXPECTED_LORA_OUTPUT = [
23
+ "SELECT count(*) FROM singer",
24
+ "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'",
25
+ "SELECT name FROM stadium WHERE stadium_id NOT IN (SELECT stadium_id FROM concert)",
26
+ ]
27
+
28
+
29
+ # visual caption
30
+ VL_QUESTION = "What is in the image?"
31
+ VL_TEST_IMAGES = [
32
+ ImageAsset("stop_sign"),
33
+ ImageAsset("cherry_blossom"),
34
+ ]
35
+ VL_EXPECTED_LORA_OUTPUT = [
36
+ 'A red STOP sign stands prominently in the foreground, with a traditional Chinese gate adorned with red lanterns and the Chinese characters "中華門" in the background, signaling the entrance to a Chinatown. A black car passes by on the street, and stone lion statues guard the entrance to the culturally rich area.', # noqa: E501
37
+ "A vibrant blue sky serves as a backdrop for the iconic Tokyo Skytree, partially obscured by the delicate pink blossoms of cherry trees in full bloom.", # noqa: E501
38
+ ]
39
+
40
+ TOKENIZER = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)
41
+
42
+
43
+ def _assert_exact_outputs(
44
+ generated_texts: list[str], expected_outputs: list[str]
45
+ ) -> None:
46
+ assert generated_texts == expected_outputs
47
+
48
+
49
+ def _assert_prefix_outputs(
50
+ generated_texts: list[str],
51
+ expected_outputs: list[str],
52
+ ) -> None:
53
+ assert len(generated_texts) == len(expected_outputs)
54
+ for generated_text, expected_text in zip(generated_texts, expected_outputs):
55
+ assert expected_text.startswith(generated_text), (
56
+ f"Generated {generated_text!r} is not a prefix of expected "
57
+ f"{expected_text!r}"
58
+ )
59
+
60
+
61
+ def _run_text_lora_sample(
62
+ llm: vllm.LLM,
63
+ lora_path: str,
64
+ lora_id: int,
65
+ ) -> list[str]:
66
+ prompts = [
67
+ TEXT_PROMPT_TEMPLATE.format(query="How many singers do we have?"),
68
+ TEXT_PROMPT_TEMPLATE.format(
69
+ query=(
70
+ "What is the average, minimum, and maximum "
71
+ "age of all singers from France?"
72
+ )
73
+ ),
74
+ TEXT_PROMPT_TEMPLATE.format(
75
+ query="What are the names of the stadiums without any concerts?"
76
+ ),
77
+ ]
78
+ input_templates = []
79
+ for prompt_text in prompts:
80
+ messages = [{"role": "user", "content": prompt_text}]
81
+ prompt = TOKENIZER.apply_chat_template(
82
+ messages,
83
+ tokenize=False,
84
+ add_generation_prompt=True,
85
+ enable_thinking=False, # disable thinking
86
+ )
87
+ input_templates.append(prompt)
88
+
89
+ outputs = llm.generate(
90
+ input_templates,
91
+ vllm.SamplingParams(temperature=0, max_tokens=512),
92
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path),
93
+ )
94
+
95
+ generated_texts: list[str] = []
96
+ for output in outputs:
97
+ generated_text = output.outputs[0].text.strip()
98
+ generated_texts.append(generated_text)
99
+ print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
100
+ return generated_texts
101
+
102
+
103
+ def _run_vl_lora_sample(
104
+ llm: vllm.LLM,
105
+ lora_path: str | None = None,
106
+ lora_id: int = 0,
107
+ ) -> list[str]:
108
+ messages = [
109
+ {
110
+ "role": "user",
111
+ "content": [
112
+ {"type": "image"},
113
+ {"type": "text", "text": VL_QUESTION},
114
+ ],
115
+ }
116
+ ]
117
+ prompt = TOKENIZER.apply_chat_template(
118
+ messages,
119
+ tokenize=False,
120
+ add_generation_prompt=True,
121
+ enable_thinking=False,
122
+ )
123
+ prompts = [
124
+ {
125
+ "prompt": prompt,
126
+ "multi_modal_data": {"image": asset.pil_image},
127
+ }
128
+ for asset in VL_TEST_IMAGES
129
+ ]
130
+ outputs = llm.generate(
131
+ prompts,
132
+ vllm.SamplingParams(temperature=0, max_tokens=128),
133
+ lora_request=(
134
+ LoRARequest(str(lora_id), lora_id, lora_path)
135
+ if lora_path is not None
136
+ else None
137
+ ),
138
+ )
139
+
140
+ generated_texts: list[str] = []
141
+ for output in outputs:
142
+ generated_text = output.outputs[0].text.strip()
143
+ generated_texts.append(generated_text)
144
+ print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
145
+ return generated_texts
146
+
147
+
148
+ def _build_text_prompts() -> list[str]:
149
+ prompts = [
150
+ TEXT_PROMPT_TEMPLATE.format(query="How many singers do we have?"),
151
+ TEXT_PROMPT_TEMPLATE.format(
152
+ query=(
153
+ "What is the average, minimum, and maximum "
154
+ "age of all singers from France?"
155
+ )
156
+ ),
157
+ TEXT_PROMPT_TEMPLATE.format(
158
+ query="What are the names of the stadiums without any concerts?"
159
+ ),
160
+ ]
161
+ input_templates = []
162
+ for prompt_text in prompts:
163
+ messages = [{"role": "user", "content": prompt_text}]
164
+ prompt = TOKENIZER.apply_chat_template(
165
+ messages,
166
+ tokenize=False,
167
+ add_generation_prompt=True,
168
+ enable_thinking=False,
169
+ )
170
+ input_templates.append(prompt)
171
+ return input_templates
172
+
173
+
174
+ def _build_vl_prompts() -> list[dict]:
175
+ messages = [
176
+ {
177
+ "role": "user",
178
+ "content": [
179
+ {"type": "image"},
180
+ {"type": "text", "text": VL_QUESTION},
181
+ ],
182
+ }
183
+ ]
184
+ prompt = TOKENIZER.apply_chat_template(
185
+ messages,
186
+ tokenize=False,
187
+ add_generation_prompt=True,
188
+ enable_thinking=False,
189
+ )
190
+ return [
191
+ {
192
+ "prompt": prompt,
193
+ "multi_modal_data": {"image": asset.pil_image},
194
+ }
195
+ for asset in VL_TEST_IMAGES
196
+ ]
197
+
198
+
199
+ def _run_mixed_lora_sample(
200
+ llm: vllm.LLM,
201
+ text_lora_path: str,
202
+ vl_lora_path: str,
203
+ text_lora_id: int,
204
+ vl_lora_id: int,
205
+ ) -> list[str]:
206
+ text_prompts = _build_text_prompts()[:2]
207
+ vl_prompts = _build_vl_prompts()
208
+ prompts = [
209
+ text_prompts[0],
210
+ vl_prompts[0],
211
+ text_prompts[1],
212
+ vl_prompts[1],
213
+ ]
214
+ lora_requests = [
215
+ LoRARequest("qwen35-text", text_lora_id, text_lora_path),
216
+ LoRARequest("qwen35-vl", vl_lora_id, vl_lora_path),
217
+ LoRARequest("qwen35-text", text_lora_id, text_lora_path),
218
+ LoRARequest("qwen35-vl", vl_lora_id, vl_lora_path),
219
+ ]
220
+ outputs = llm.generate(
221
+ prompts,
222
+ vllm.SamplingParams(temperature=0, max_tokens=256),
223
+ lora_request=lora_requests,
224
+ )
225
+
226
+ generated_texts: list[str] = []
227
+ for output in outputs:
228
+ generated_text = output.outputs[0].text.strip()
229
+ generated_texts.append(generated_text)
230
+ print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
231
+ return generated_texts
232
+
233
+
234
+ def _run_mixed_lora_and_base_sample(
235
+ llm: vllm.LLM,
236
+ text_lora_path: str,
237
+ vl_lora_path: str,
238
+ text_lora_id: int,
239
+ vl_lora_id: int,
240
+ ) -> list[str]:
241
+ text_prompt = _build_text_prompts()[0]
242
+ vl_prompt = _build_vl_prompts()[0]
243
+ prompts = [
244
+ text_prompt,
245
+ vl_prompt,
246
+ text_prompt,
247
+ vl_prompt,
248
+ ]
249
+ lora_requests = [
250
+ LoRARequest("qwen35-text", text_lora_id, text_lora_path),
251
+ LoRARequest("qwen35-vl", vl_lora_id, vl_lora_path),
252
+ None,
253
+ None,
254
+ ]
255
+ outputs = llm.generate(
256
+ prompts,
257
+ vllm.SamplingParams(temperature=0, max_tokens=256),
258
+ lora_request=lora_requests,
259
+ )
260
+
261
+ generated_texts: list[str] = []
262
+ for output in outputs:
263
+ generated_text = output.outputs[0].text.strip()
264
+ generated_texts.append(generated_text)
265
+ print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
266
+ return generated_texts
267
+
268
+
269
+ def _assert_qwen35_text_vl_and_mixed_lora(
270
+ llm: vllm.LLM,
271
+ qwen35_text_lora_files: str,
272
+ qwen35_vl_lora_files: str,
273
+ ) -> None:
274
+ generated_texts = _run_text_lora_sample(
275
+ llm,
276
+ qwen35_text_lora_files,
277
+ TEXT_LORA_ID,
278
+ )
279
+
280
+ _assert_exact_outputs(generated_texts, TEXT_EXPECTED_LORA_OUTPUT)
281
+
282
+ generated_texts = _run_vl_lora_sample(
283
+ llm,
284
+ qwen35_vl_lora_files,
285
+ VL_LORA_ID,
286
+ )
287
+ _assert_prefix_outputs(generated_texts, VL_EXPECTED_LORA_OUTPUT)
288
+
289
+ generated_texts = _run_mixed_lora_sample(
290
+ llm,
291
+ qwen35_text_lora_files,
292
+ qwen35_vl_lora_files,
293
+ text_lora_id=TEXT_LORA_ID,
294
+ vl_lora_id=VL_LORA_ID,
295
+ )
296
+ assert generated_texts[0] == TEXT_EXPECTED_LORA_OUTPUT[0]
297
+ assert generated_texts[2] == TEXT_EXPECTED_LORA_OUTPUT[1]
298
+ _assert_prefix_outputs([generated_texts[1]], [VL_EXPECTED_LORA_OUTPUT[0]])
299
+ _assert_prefix_outputs([generated_texts[3]], [VL_EXPECTED_LORA_OUTPUT[1]])
300
+
301
+ generated_texts = _run_mixed_lora_and_base_sample(
302
+ llm,
303
+ qwen35_text_lora_files,
304
+ qwen35_vl_lora_files,
305
+ text_lora_id=TEXT_LORA_ID,
306
+ vl_lora_id=VL_LORA_ID,
307
+ )
308
+ assert generated_texts[0] == TEXT_EXPECTED_LORA_OUTPUT[0]
309
+ _assert_prefix_outputs([generated_texts[1]], [VL_EXPECTED_LORA_OUTPUT[0]])
310
+ assert generated_texts[2] != TEXT_EXPECTED_LORA_OUTPUT[0]
311
+ assert not VL_EXPECTED_LORA_OUTPUT[0].startswith(generated_texts[3]), (
312
+ "Non-LoRA vision output unexpectedly matches the LoRA expectation."
313
+ )
314
+
315
+
316
+ @pytest.mark.skipif(
317
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
318
+ )
319
+ @create_new_process_for_each_test()
320
+ def test_qwen35_text_lora(
321
+ qwen35_text_lora_files, qwen35_vl_lora_files, maybe_enable_lora_dual_stream
322
+ ):
323
+ llm = vllm.LLM(
324
+ model=MODEL_PATH,
325
+ max_model_len=4096,
326
+ enable_lora=True,
327
+ max_loras=2,
328
+ max_num_seqs=4,
329
+ max_lora_rank=8,
330
+ enforce_eager=True,
331
+ trust_remote_code=True,
332
+ enable_tower_connector_lora=True,
333
+ mm_processor_cache_gb=0,
334
+ limit_mm_per_prompt={"image": 1},
335
+ )
336
+
337
+ _assert_qwen35_text_vl_and_mixed_lora(
338
+ llm,
339
+ qwen35_text_lora_files,
340
+ qwen35_vl_lora_files,
341
+ )
342
+
343
+
344
+ @multi_gpu_test(num_gpus=4)
345
+ def test_qwen35_text_lora_tp4(
346
+ qwen35_text_lora_files, qwen35_vl_lora_files, maybe_enable_lora_dual_stream
347
+ ):
348
+ llm = vllm.LLM(
349
+ model=MODEL_PATH,
350
+ max_model_len=4096,
351
+ enable_lora=True,
352
+ max_loras=2,
353
+ max_lora_rank=8,
354
+ max_num_seqs=4,
355
+ enforce_eager=True,
356
+ tensor_parallel_size=4,
357
+ trust_remote_code=True,
358
+ enable_tower_connector_lora=True,
359
+ mm_processor_cache_gb=0,
360
+ limit_mm_per_prompt={"image": 1},
361
+ compilation_config=vllm.config.CompilationConfig(
362
+ cudagraph_specialize_lora=False,
363
+ ),
364
+ )
365
+
366
+ _assert_qwen35_text_vl_and_mixed_lora(
367
+ llm,
368
+ qwen35_text_lora_files,
369
+ qwen35_vl_lora_files,
370
+ )
asian_ai/performance/vLLM/tests/lora/test_qwen36_moe_lora.py ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import pytest
5
+
6
+ import vllm
7
+ import vllm.config
8
+ from vllm.assets.image import ImageAsset
9
+ from vllm.lora.request import LoRARequest
10
+
11
+ from ..utils import multi_gpu_test
12
+
13
+ MODEL_PATH = "Qwen/Qwen3.6-35B-A3B"
14
+
15
+ LORA_2D_ID = 1
16
+ LORA_3D_ID = 2
17
+
18
+ PROMPT_TEMPLATE = """<|im_start|>user
19
+ <|vision_start|><|image_pad|><|vision_end|>What is in the image?<|im_end|>
20
+ <|im_start|>assistant
21
+ <think>
22
+
23
+ </think>
24
+ """
25
+
26
+ # Visual captioning prompts: each image will be paired with one LoRA in the
27
+ # mixed-batch case so we can check per-prompt routing.
28
+ VL_TEST_IMAGES = [
29
+ ImageAsset("stop_sign"),
30
+ ImageAsset("cherry_blossom"),
31
+ ]
32
+
33
+
34
+ def _build_prompts() -> list[dict]:
35
+ return [
36
+ {
37
+ "prompt": PROMPT_TEMPLATE,
38
+ "multi_modal_data": {"image": asset.pil_image},
39
+ }
40
+ for asset in VL_TEST_IMAGES
41
+ ]
42
+
43
+
44
+ def _generate(llm: vllm.LLM, lora_request) -> list[str]:
45
+ outputs = llm.generate(
46
+ _build_prompts(),
47
+ vllm.SamplingParams(temperature=0, max_tokens=128),
48
+ lora_request=lora_request,
49
+ )
50
+ return [out.outputs[0].text.strip() for out in outputs]
51
+
52
+
53
+ def _run_mixed_2d_3d_lora_test(
54
+ lora_2d_files: str,
55
+ lora_3d_files: str,
56
+ tensor_parallel_size: int,
57
+ fully_sharded_loras: bool,
58
+ ) -> None:
59
+ llm = vllm.LLM(
60
+ model=MODEL_PATH,
61
+ max_model_len=4096,
62
+ enable_lora=True,
63
+ enable_mixed_moe_lora_format=True,
64
+ max_loras=2,
65
+ max_lora_rank=8,
66
+ max_num_seqs=4,
67
+ enforce_eager=True,
68
+ tensor_parallel_size=tensor_parallel_size,
69
+ enable_expert_parallel=not fully_sharded_loras,
70
+ fully_sharded_loras=fully_sharded_loras,
71
+ trust_remote_code=True,
72
+ enable_tower_connector_lora=True,
73
+ mm_processor_cache_gb=0,
74
+ limit_mm_per_prompt={"image": 1},
75
+ compilation_config=vllm.config.CompilationConfig(
76
+ cudagraph_specialize_lora=False,
77
+ ),
78
+ )
79
+
80
+ lora_2d = LoRARequest(
81
+ "lora_2d",
82
+ LORA_2D_ID,
83
+ lora_2d_files,
84
+ is_3d_lora_weight=False,
85
+ )
86
+ lora_3d = LoRARequest(
87
+ "lora_3d",
88
+ LORA_3D_ID,
89
+ lora_3d_files,
90
+ is_3d_lora_weight=True,
91
+ )
92
+
93
+ # Reference: each adapter alone over both prompts.
94
+ outputs_2d_alone = _generate(llm, lora_2d)
95
+ outputs_3d_alone = _generate(llm, lora_3d)
96
+
97
+ assert len(outputs_2d_alone) == len(VL_TEST_IMAGES)
98
+ assert len(outputs_3d_alone) == len(VL_TEST_IMAGES)
99
+ for text in outputs_2d_alone + outputs_3d_alone:
100
+ assert text, "Empty output from single-adapter LoRA generation"
101
+
102
+ # Mixed batch: prompt 0 uses the 2D adapter, prompt 1 uses the 3D
103
+ # adapter. Per-prompt outputs must match the standalone runs.
104
+ mixed_outputs = _generate(llm, [lora_2d, lora_3d])
105
+
106
+ assert mixed_outputs[0] == outputs_2d_alone[0], (
107
+ f"Mixed-batch 2D output {mixed_outputs[0]!r} does not match "
108
+ f"standalone 2D output {outputs_2d_alone[0]!r}"
109
+ )
110
+ assert mixed_outputs[1] == outputs_3d_alone[1], (
111
+ f"Mixed-batch 3D output {mixed_outputs[1]!r} does not match "
112
+ f"standalone 3D output {outputs_3d_alone[1]!r}"
113
+ )
114
+
115
+ # Reverse assignment: neither adapter should be silently aliased.
116
+ swapped_outputs = _generate(llm, [lora_3d, lora_2d])
117
+ assert swapped_outputs[0] == outputs_3d_alone[0], (
118
+ f"Swapped-batch 3D output {swapped_outputs[0]!r} does not match "
119
+ f"standalone 3D output {outputs_3d_alone[0]!r}"
120
+ )
121
+ assert swapped_outputs[1] == outputs_2d_alone[1], (
122
+ f"Swapped-batch 2D output {swapped_outputs[1]!r} does not match "
123
+ f"standalone 2D output {outputs_2d_alone[1]!r}"
124
+ )
125
+
126
+
127
+ @pytest.mark.skip(reason="This model is too big, so skip this test temporarily.")
128
+ @pytest.mark.parametrize("fully_sharded_loras", [False, True])
129
+ @multi_gpu_test(num_gpus=2)
130
+ def test_qwen36_moe_mixed_2d_3d_lora_tp2(
131
+ qwen36_moe_2d_lora_files,
132
+ qwen36_moe_3d_lora_files,
133
+ fully_sharded_loras,
134
+ ):
135
+ _run_mixed_2d_3d_lora_test(
136
+ lora_2d_files=qwen36_moe_2d_lora_files,
137
+ lora_3d_files=qwen36_moe_3d_lora_files,
138
+ tensor_parallel_size=2,
139
+ fully_sharded_loras=fully_sharded_loras,
140
+ )
141
+
142
+
143
+ @pytest.mark.skip(reason="This model is too big, so skip this test temporarily.")
144
+ @pytest.mark.parametrize("fully_sharded_loras", [False, True])
145
+ @multi_gpu_test(num_gpus=4)
146
+ def test_qwen36_moe_mixed_2d_3d_lora_tp4(
147
+ qwen36_moe_2d_lora_files,
148
+ qwen36_moe_3d_lora_files,
149
+ fully_sharded_loras,
150
+ ):
151
+ _run_mixed_2d_3d_lora_test(
152
+ lora_2d_files=qwen36_moe_2d_lora_files,
153
+ lora_3d_files=qwen36_moe_3d_lora_files,
154
+ tensor_parallel_size=4,
155
+ fully_sharded_loras=fully_sharded_loras,
156
+ )
asian_ai/performance/vLLM/tests/lora/test_qwen3_unembed.py ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ """
4
+ Tests for Qwen3 unembed LoRA support.
5
+
6
+ This test creates synthetic LoRA weights that include lm_head (output embedding)
7
+ to verify that Qwen3 properly supports LoRA on the unembed/lm_head layer.
8
+ """
9
+
10
+ import json
11
+ import os
12
+ import tempfile
13
+
14
+ import numpy as np
15
+ import torch
16
+ from safetensors.torch import save_file
17
+
18
+ from vllm import LLM, SamplingParams
19
+ from vllm.lora.request import LoRARequest
20
+
21
+ MODEL_PATH = "Qwen/Qwen3-0.6B"
22
+ HIDDEN_SIZE = 1024
23
+ VOCAB_SIZE = 151936
24
+
25
+
26
+ def create_qwen3_lora_with_lm_head(save_dir: str, rank: int = 8) -> None:
27
+ """Create synthetic Qwen3 LoRA weights with lm_head."""
28
+ lora_weights = {}
29
+ for module in ["q_proj", "v_proj"]:
30
+ lora_A = torch.from_numpy(
31
+ np.random.randn(rank, HIDDEN_SIZE).astype(np.float16) * 0.01
32
+ )
33
+ lora_B = torch.zeros(HIDDEN_SIZE, rank, dtype=torch.float16)
34
+ key_prefix = f"base_model.model.model.layers.0.self_attn.{module}"
35
+ lora_weights[f"{key_prefix}.lora_A.weight"] = lora_A
36
+ lora_weights[f"{key_prefix}.lora_B.weight"] = lora_B
37
+
38
+ # lm_head LoRA weights
39
+ lora_weights["base_model.model.lm_head.lora_A.weight"] = torch.from_numpy(
40
+ np.random.randn(rank, HIDDEN_SIZE).astype(np.float16) * 0.01
41
+ )
42
+ lora_weights["base_model.model.lm_head.lora_B.weight"] = torch.zeros(
43
+ VOCAB_SIZE, rank, dtype=torch.float16
44
+ )
45
+
46
+ adapter_config = {
47
+ "peft_type": "LORA",
48
+ "base_model_name_or_path": MODEL_PATH,
49
+ "task_type": "CAUSAL_LM",
50
+ "inference_mode": True,
51
+ "r": rank,
52
+ "lora_alpha": rank * 2,
53
+ "lora_dropout": 0.0,
54
+ "bias": "none",
55
+ "target_modules": ["q_proj", "v_proj", "lm_head"],
56
+ }
57
+
58
+ os.makedirs(save_dir, exist_ok=True)
59
+ with open(os.path.join(save_dir, "adapter_config.json"), "w") as f:
60
+ json.dump(adapter_config, f)
61
+ save_file(lora_weights, os.path.join(save_dir, "adapter_model.safetensors"))
62
+
63
+
64
+ def test_qwen3_unembed_lora():
65
+ """Verify Qwen3 can load and generate with LoRA adapters with lm_head."""
66
+ with tempfile.TemporaryDirectory() as tmpdir:
67
+ # Initialize engine first (before creating torch tensors)
68
+ llm = LLM(
69
+ model=MODEL_PATH,
70
+ enable_lora=True,
71
+ max_loras=4,
72
+ max_lora_rank=8,
73
+ max_model_len=128,
74
+ gpu_memory_utilization=0.8,
75
+ enforce_eager=True,
76
+ )
77
+
78
+ # Create LoRA weights after engine init
79
+ create_qwen3_lora_with_lm_head(tmpdir, rank=8)
80
+
81
+ lora_request = LoRARequest("lm_head_lora", 1, tmpdir)
82
+ llm.llm_engine.add_lora(lora_request)
83
+
84
+ assert 1 in llm.llm_engine.list_loras(), "lm_head LoRA should be loaded"
85
+
86
+ # Test generation
87
+ sampling_params = SamplingParams(temperature=0, max_tokens=32)
88
+ prompts = ["Hello, my name is"]
89
+
90
+ # Generate with base model (no LoRA)
91
+ base_outputs = llm.generate(prompts, sampling_params, use_tqdm=False)
92
+ assert len(base_outputs) == 1
93
+ assert len(base_outputs[0].outputs[0].text) > 0
94
+
95
+ # Generate with lm_head LoRA
96
+ lora_outputs = llm.generate(
97
+ prompts, sampling_params, lora_request=lora_request, use_tqdm=False
98
+ )
99
+ assert len(lora_outputs) == 1
100
+ assert len(lora_outputs[0].outputs[0].text) > 0
asian_ai/performance/vLLM/tests/lora/test_qwen3_with_multi_loras.py ADDED
@@ -0,0 +1,306 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ """
4
+ This script contains:
5
+ 1. test multi loras service with tp >= 2
6
+ 2. test multi loras request
7
+ """
8
+
9
+ import os
10
+
11
+ import pytest
12
+
13
+ from tests.utils import multi_gpu_test
14
+ from vllm import LLM, SamplingParams
15
+ from vllm.lora.request import LoRARequest
16
+
17
+ MODEL_PATH = "Qwen/Qwen3-0.6B"
18
+ LORA_NAME_PATH_MAP = {
19
+ "Alice": "charent/self_cognition_Alice",
20
+ "Bob": "charent/self_cognition_Bob",
21
+ "Cat": "charent/self_cognition_Bob", # same as Bob
22
+ }
23
+
24
+ LORA_NAME_ID_MAP = {}
25
+ INCREASE_LORA_ID = 0
26
+ LORA_RANK = 8
27
+
28
+ LORA_TEST_PROMPTS = ["What is GitHub?", "Hi, tell me about you"]
29
+ LORA_TEST_EXPECTED = [
30
+ "GitHub is an open-source platform that provides a way to manage and develop software projects. It allows developers to store and manage code, collaborate on projects, and automate tasks.", # noqa: E501
31
+ "I am Alice, an AI assistant developed by GitHub/Charent.",
32
+ ]
33
+
34
+
35
+ def format_chatml_messages(
36
+ prompt: str, system_prompt: str = "You are a helpful assistant."
37
+ ) -> list[dict[str, str]]:
38
+ return [
39
+ {"role": "system", "content": system_prompt},
40
+ {"role": "user", "content": prompt},
41
+ ]
42
+
43
+
44
+ @pytest.fixture(autouse=True)
45
+ def set_mrv2_env():
46
+ original = os.environ.get("VLLM_USE_V2_MODEL_RUNNER", "0")
47
+ os.environ["VLLM_USE_V2_MODEL_RUNNER"] = "1"
48
+ yield
49
+
50
+ if original is None:
51
+ os.environ.pop("VLLM_USE_V2_MODEL_RUNNER", None)
52
+ else:
53
+ os.environ["VLLM_USE_V2_MODEL_RUNNER"] = original
54
+
55
+
56
+ def make_add_lora_request(name: str, path: str):
57
+ global INCREASE_LORA_ID, LORA_NAME_ID_MAP
58
+
59
+ INCREASE_LORA_ID += 1
60
+ LORA_NAME_ID_MAP[name] = INCREASE_LORA_ID
61
+
62
+ return LoRARequest(
63
+ lora_name=name,
64
+ lora_int_id=INCREASE_LORA_ID,
65
+ lora_path=path,
66
+ )
67
+
68
+
69
+ @multi_gpu_test(num_gpus=2)
70
+ def test_multi_loras_with_tp_sync():
71
+ llm = LLM(
72
+ model=MODEL_PATH,
73
+ enable_lora=True,
74
+ max_loras=2, # ensure max_loras < max_cpu_loras
75
+ max_lora_rank=LORA_RANK,
76
+ max_model_len=512,
77
+ gpu_memory_utilization=0.5,
78
+ tensor_parallel_size=2, # ensure tp >= 2
79
+ max_cpu_loras=4, # ensure max_cpu_loras >= 2
80
+ )
81
+
82
+ def run_check_lora(fn, args, expected: list):
83
+ fn(args)
84
+ assert set(llm.llm_engine.list_loras()) == set(expected)
85
+
86
+ # simulate add loras with CLI args
87
+ # likes: `--lora-modules Alice=/path/to/Alice Bob=/path/to/Bob`
88
+ run_check_lora(
89
+ llm.llm_engine.add_lora,
90
+ make_add_lora_request("Alice", LORA_NAME_PATH_MAP["Alice"]),
91
+ [1],
92
+ )
93
+ run_check_lora(
94
+ llm.llm_engine.add_lora,
95
+ make_add_lora_request("Bob", LORA_NAME_PATH_MAP["Bob"]),
96
+ [1, 2],
97
+ )
98
+ run_check_lora(
99
+ llm.llm_engine.add_lora,
100
+ make_add_lora_request("Cat", LORA_NAME_PATH_MAP["Cat"]),
101
+ [1, 2, 3],
102
+ )
103
+
104
+ # set temperature = 0 for greedy search
105
+ sampling_params = SamplingParams(temperature=0, max_tokens=64)
106
+
107
+ def call_llm_get_outputs(prompt: str, lora_name: str):
108
+ lora_request = LoRARequest(
109
+ lora_name=lora_name,
110
+ lora_int_id=LORA_NAME_ID_MAP[lora_name],
111
+ lora_path=LORA_NAME_PATH_MAP[lora_name],
112
+ )
113
+ messages = format_chatml_messages(prompt)
114
+ outputs = llm.chat(
115
+ [messages],
116
+ sampling_params,
117
+ chat_template_kwargs={
118
+ "enable_thinking": False
119
+ }, # for those loras, ensure enable_thinking=False
120
+ lora_request=lora_request,
121
+ use_tqdm=False,
122
+ )
123
+ output_text = outputs[0].outputs[0].text
124
+ return output_text
125
+
126
+ def reload_lora(name: str):
127
+ """
128
+ reload a lora to simulate the case:
129
+ setting `VLLM_ALLOW_RUNTIME_LORA_UPDATING=true`
130
+ for dynamic lora loading and unloading
131
+ """
132
+ remove_lora_response = llm.llm_engine.remove_lora(
133
+ lora_id=LORA_NAME_ID_MAP[name]
134
+ )
135
+
136
+ add_lora_response = llm.llm_engine.add_lora(
137
+ make_add_lora_request(name, LORA_NAME_PATH_MAP[name])
138
+ )
139
+
140
+ print(f"{remove_lora_response=}, {add_lora_response=}")
141
+
142
+ def check_outputs(outputs: str, expected: str):
143
+ print(f"{prompt=}.\n{expected_output=}\n{output_text=}")
144
+ print("\n----------------------------\n")
145
+ assert outputs == expected
146
+
147
+ for prompt, expected_output in zip(LORA_TEST_PROMPTS, LORA_TEST_EXPECTED):
148
+ output_text = call_llm_get_outputs(prompt, "Alice")
149
+ check_outputs(output_text, expected_output)
150
+
151
+ # call Bob, ignore what it is output
152
+ call_llm_get_outputs(prompt, "Bob")
153
+ print("After call Bob:")
154
+
155
+ # call Alice
156
+ output_text = call_llm_get_outputs(prompt, "Alice")
157
+ check_outputs(output_text, expected_output)
158
+
159
+ # reload Bob Lora
160
+ reload_lora("Bob")
161
+ print("After reload Bob:")
162
+
163
+ # call Alice
164
+ output_text = call_llm_get_outputs(prompt, "Alice")
165
+ check_outputs(output_text, expected_output)
166
+
167
+ # reload Alice Lora
168
+ reload_lora("Alice")
169
+ print("After reload Alice:")
170
+
171
+ output_text = call_llm_get_outputs(prompt, "Alice")
172
+ check_outputs(output_text, expected_output)
173
+
174
+
175
+ def test_multiple_lora_requests():
176
+ llm = LLM(
177
+ model=MODEL_PATH,
178
+ enable_lora=True,
179
+ max_loras=4,
180
+ max_lora_rank=LORA_RANK,
181
+ max_model_len=512,
182
+ gpu_memory_utilization=0.5,
183
+ )
184
+ PROMPTS = ["Hello, my name is"] * 2
185
+ LORA_NAME = "Alice"
186
+ lora_request = [
187
+ LoRARequest(LORA_NAME + str(idx), idx + 1, LORA_NAME_PATH_MAP[LORA_NAME])
188
+ for idx in range(len(PROMPTS))
189
+ ]
190
+ # Multiple SamplingParams should be matched with each prompt
191
+ outputs = llm.generate(PROMPTS, lora_request=lora_request)
192
+ assert len(PROMPTS) == len(outputs)
193
+
194
+ # Exception raised, if the size of params does not match the size of prompts
195
+ with pytest.raises(ValueError):
196
+ outputs = llm.generate(PROMPTS, lora_request=lora_request[:1])
197
+
198
+ # Single LoRARequest should be applied to every prompt
199
+ single_lora_request = lora_request[0]
200
+ outputs = llm.generate(PROMPTS, lora_request=single_lora_request)
201
+ assert len(PROMPTS) == len(outputs)
202
+
203
+
204
+ def test_load_inplace_offline_reload(
205
+ qwen3_meowing_lora_files: str, qwen3_woofing_lora_files: str
206
+ ) -> None:
207
+ """
208
+ Test that load_inplace=True allows reloading LoRA adapters with the same ID
209
+ in offline mode (using LLM class directly).
210
+ """
211
+ llm = LLM(
212
+ model=MODEL_PATH,
213
+ enable_lora=True,
214
+ max_loras=2,
215
+ max_lora_rank=LORA_RANK,
216
+ max_model_len=512,
217
+ gpu_memory_utilization=0.5,
218
+ )
219
+ adapter_id = 1
220
+ messages = format_chatml_messages(
221
+ "Make your favorite animal noise.",
222
+ system_prompt="Follow the instructions to make animal noises",
223
+ )
224
+ sampling_params = SamplingParams(temperature=0, max_tokens=10)
225
+
226
+ # Load meowing LoRA with load_inplace=True
227
+ meowing_request = LoRARequest(
228
+ lora_name="test-adapter",
229
+ lora_int_id=adapter_id,
230
+ lora_path=qwen3_meowing_lora_files,
231
+ )
232
+
233
+ outputs = llm.chat([messages], sampling_params, lora_request=meowing_request)
234
+ first_output = outputs[0].outputs[0].text.strip()
235
+ assert "Meow Meow Meow" in first_output, (
236
+ f"Expected meowing output, got: {first_output}"
237
+ )
238
+
239
+ # Reload with woofing LoRA (same ID, different weights, load_inplace=True)
240
+ woofing_request = LoRARequest(
241
+ lora_name="test-adapter-woof",
242
+ lora_int_id=adapter_id, # Same ID
243
+ lora_path=qwen3_woofing_lora_files, # Different weights
244
+ load_inplace=True, # Force reload
245
+ )
246
+
247
+ outputs = llm.chat([messages], sampling_params, lora_request=woofing_request)
248
+ second_output = outputs[0].outputs[0].text.strip()
249
+ assert "Woof Woof Woof" in second_output, (
250
+ f"Expected woofing output, got: {second_output}"
251
+ )
252
+
253
+
254
+ def test_load_inplace_false_no_reload(
255
+ qwen3_meowing_lora_files: str, qwen3_woofing_lora_files: str
256
+ ) -> None:
257
+ """
258
+ Test that load_inplace=False prevents reloading when an adapter
259
+ with the same ID already exists.
260
+ """
261
+ llm = LLM(
262
+ model=MODEL_PATH,
263
+ enable_lora=True,
264
+ max_loras=2,
265
+ max_lora_rank=LORA_RANK,
266
+ max_model_len=512,
267
+ gpu_memory_utilization=0.5,
268
+ )
269
+ adapter_id = 2
270
+ messages = format_chatml_messages(
271
+ "Make your favorite animal noise.",
272
+ system_prompt="Follow the instructions to make animal noises",
273
+ )
274
+ sampling_params = SamplingParams(temperature=0, max_tokens=10)
275
+
276
+ # Load meowing LoRA first with load_inplace=True
277
+ meowing_request_initial = LoRARequest(
278
+ lora_name="test-adapter-2",
279
+ lora_int_id=adapter_id,
280
+ lora_path=qwen3_meowing_lora_files,
281
+ )
282
+
283
+ outputs = llm.chat(
284
+ [messages], sampling_params, lora_request=meowing_request_initial
285
+ )
286
+ first_output = outputs[0].outputs[0].text.strip()
287
+ assert "Meow Meow Meow" in first_output, (
288
+ f"Expected meowing output, got: {first_output}"
289
+ )
290
+
291
+ # Try to load woofing LoRA with same ID but load_inplace=False
292
+ # This should NOT reload (adapter 2 already exists)
293
+ woofing_request_no_reload = LoRARequest(
294
+ lora_name="test-adapter-2-woof",
295
+ lora_int_id=adapter_id, # Same ID
296
+ lora_path=qwen3_woofing_lora_files,
297
+ )
298
+
299
+ outputs = llm.chat(
300
+ [messages], sampling_params, lora_request=woofing_request_no_reload
301
+ )
302
+ second_output = outputs[0].outputs[0].text.strip()
303
+ # Should still get meowing output because it didn't reload
304
+ assert "Meow Meow Meow" in second_output, (
305
+ f"Expected meowing output (no reload), got: {second_output}"
306
+ )
asian_ai/performance/vLLM/tests/lora/test_qwen3moe_tp.py ADDED
@@ -0,0 +1,122 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+ # NOTE To avoid overloading the CI pipeline, this test script will not
6
+ # be triggered on CI and is primarily intended for local testing and verification.
7
+
8
+ import pytest
9
+
10
+ import vllm
11
+ from vllm.lora.request import LoRARequest
12
+
13
+ from ..utils import multi_gpu_test
14
+
15
+ MODEL_PATH = "Qwen/Qwen3-30B-A3B"
16
+
17
+ PROMPT_TEMPLATE = """<|im_start|>user
18
+ I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.
19
+ "
20
+ ##Instruction:
21
+ candidate_poll contains tables such as candidate, people. Table candidate has columns such as Candidate_ID, People_ID, Poll_Source, Date, Support_rate, Consider_rate, Oppose_rate, Unsure_rate. Candidate_ID is the primary key.
22
+ Table people has columns such as People_ID, Sex, Name, Date_of_Birth, Height, Weight. People_ID is the primary key.
23
+ The People_ID of candidate is the foreign key of People_ID of people.
24
+
25
+
26
+ ###Input:
27
+ {context}
28
+
29
+ ###Response:<|im_end|>
30
+ <|im_start|>assistant""" # noqa: E501
31
+
32
+ EXPECTED_LORA_OUTPUT = [
33
+ "<think>\n\n</think>\n\nSELECT count(*) FROM candidate",
34
+ "<think>\n\n</think>\n\nSELECT count(*) FROM candidate",
35
+ "<think>\n\n</think>\n\nSELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1", # noqa: E501
36
+ "<think>\n\n</think>\n\nSELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1", # noqa: E501
37
+ ]
38
+
39
+
40
+ def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
41
+ prompts = [
42
+ PROMPT_TEMPLATE.format(context="How many candidates are there?"),
43
+ PROMPT_TEMPLATE.format(context="Count the number of candidates."),
44
+ PROMPT_TEMPLATE.format(
45
+ context="Which poll resource provided the most number of candidate information?" # noqa: E501
46
+ ),
47
+ PROMPT_TEMPLATE.format(
48
+ context="Return the poll resource associated with the most candidates."
49
+ ),
50
+ ]
51
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=64)
52
+ outputs = llm.generate(
53
+ prompts,
54
+ sampling_params,
55
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
56
+ )
57
+ # Print the outputs.
58
+ generated_texts: list[str] = []
59
+ for output in outputs:
60
+ prompt = output.prompt
61
+ generated_text = output.outputs[0].text.strip()
62
+ generated_texts.append(generated_text)
63
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
64
+
65
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
66
+ assert generated_texts[i].startswith(EXPECTED_LORA_OUTPUT[i])
67
+
68
+
69
+ def test_qwen3moe_lora(qwen3moe_lora_files):
70
+ # We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
71
+ # Otherwise, the lora-test will fail due to CUDA OOM.
72
+ llm = vllm.LLM(
73
+ MODEL_PATH,
74
+ max_model_len=1024,
75
+ enable_lora=True,
76
+ max_loras=4,
77
+ enforce_eager=True,
78
+ trust_remote_code=True,
79
+ enable_chunked_prefill=True,
80
+ enable_mixed_moe_lora_format=True,
81
+ )
82
+
83
+ generate_and_test(llm, qwen3moe_lora_files, lora_id=1)
84
+ generate_and_test(llm, qwen3moe_lora_files, lora_id=2)
85
+
86
+
87
+ @multi_gpu_test(num_gpus=2)
88
+ @pytest.mark.parametrize("ep", [False, True])
89
+ @pytest.mark.parametrize("enable_mixed_moe_lora_format", [False, True])
90
+ def test_qwen3moe_lora_tp2(ep, qwen3moe_lora_files, enable_mixed_moe_lora_format):
91
+ llm = vllm.LLM(
92
+ MODEL_PATH,
93
+ max_model_len=1024,
94
+ enable_lora=True,
95
+ max_loras=4,
96
+ trust_remote_code=True,
97
+ enable_expert_parallel=ep,
98
+ tensor_parallel_size=2,
99
+ enable_mixed_moe_lora_format=enable_mixed_moe_lora_format,
100
+ )
101
+
102
+ generate_and_test(llm, qwen3moe_lora_files, lora_id=1)
103
+ generate_and_test(llm, qwen3moe_lora_files, lora_id=2)
104
+
105
+
106
+ @multi_gpu_test(num_gpus=4)
107
+ @pytest.mark.parametrize("ep", [False, True])
108
+ @pytest.mark.parametrize("enable_mixed_moe_lora_format", [False, True])
109
+ def test_qwen3moe_lora_tp4(ep, qwen3moe_lora_files, enable_mixed_moe_lora_format):
110
+ llm = vllm.LLM(
111
+ MODEL_PATH,
112
+ max_model_len=1024,
113
+ enable_lora=True,
114
+ max_loras=4,
115
+ trust_remote_code=True,
116
+ enable_expert_parallel=ep,
117
+ tensor_parallel_size=4,
118
+ enable_mixed_moe_lora_format=enable_mixed_moe_lora_format,
119
+ )
120
+
121
+ generate_and_test(llm, qwen3moe_lora_files, lora_id=1)
122
+ generate_and_test(llm, qwen3moe_lora_files, lora_id=2)
asian_ai/performance/vLLM/tests/lora/test_qwenvl.py ADDED
@@ -0,0 +1,320 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ from dataclasses import dataclass
4
+
5
+ import pytest
6
+ from packaging.version import Version
7
+ from transformers import __version__ as TRANSFORMERS_VERSION
8
+
9
+ import vllm
10
+ from vllm.assets.image import ImageAsset
11
+ from vllm.lora.request import LoRARequest
12
+ from vllm.platforms import current_platform
13
+ from vllm.sampling_params import BeamSearchParams
14
+
15
+
16
+ @dataclass
17
+ class TestConfig:
18
+ model_path: str
19
+ lora_path: str
20
+ max_num_seqs: int = 2
21
+ max_loras: int = 2
22
+ max_lora_rank: int = 32
23
+ enable_tower_connector_lora: bool = False
24
+ max_model_len: int = 8192
25
+ gpu_memory_utilization: float = 0.85
26
+ mm_processor_kwargs: dict[str, object] | None = None
27
+ mm_processor_cache_gb: float = 4
28
+
29
+ def __post_init__(self):
30
+ if self.mm_processor_kwargs is None:
31
+ # There is a bug in transformers v4 where size is ignored by
32
+ # `Qwen2VLProcessor.__call__`
33
+ if Version(TRANSFORMERS_VERSION) < Version("5.2.0"):
34
+ self.mm_processor_kwargs = {
35
+ "min_pixels": 28 * 28,
36
+ "max_pixels": 1280 * 28 * 28,
37
+ }
38
+ else:
39
+ self.mm_processor_kwargs = {
40
+ "size": {
41
+ "shortest_edge": 28 * 28,
42
+ "longest_edge": 1280 * 28 * 28,
43
+ }
44
+ }
45
+
46
+
47
+ class Qwen2VLTester:
48
+ """Test helper for Qwen2 VL models with LoRA"""
49
+
50
+ PROMPT_TEMPLATE = (
51
+ "<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
52
+ "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
53
+ "What is in the image?<|im_end|>\n"
54
+ "<|im_start|>assistant\n"
55
+ )
56
+
57
+ def __init__(self, config: TestConfig):
58
+ self.config = config
59
+ self.llm = self._initialize_llm()
60
+
61
+ def _initialize_llm(self) -> vllm.LLM:
62
+ """Initialize the LLM with given configuration"""
63
+ return vllm.LLM(
64
+ model=self.config.model_path,
65
+ max_num_seqs=self.config.max_num_seqs,
66
+ enable_lora=True,
67
+ max_loras=self.config.max_loras,
68
+ max_lora_rank=self.config.max_lora_rank,
69
+ enable_tower_connector_lora=self.config.enable_tower_connector_lora,
70
+ trust_remote_code=True,
71
+ gpu_memory_utilization=self.config.gpu_memory_utilization,
72
+ mm_processor_kwargs=self.config.mm_processor_kwargs,
73
+ mm_processor_cache_gb=self.config.mm_processor_cache_gb,
74
+ max_model_len=self.config.max_model_len,
75
+ )
76
+
77
+ def run_test(
78
+ self,
79
+ images: list[ImageAsset],
80
+ expected_outputs: list[str],
81
+ lora_id: int | None = None,
82
+ lora_name: str | None = None,
83
+ temperature: float = 0,
84
+ max_tokens: int = 5,
85
+ ):
86
+ sampling_params = vllm.SamplingParams(
87
+ temperature=temperature,
88
+ max_tokens=max_tokens,
89
+ )
90
+ inputs = [
91
+ {
92
+ "prompt": self.PROMPT_TEMPLATE,
93
+ "multi_modal_data": {"image": asset.pil_image},
94
+ }
95
+ for asset in images
96
+ ]
97
+
98
+ lora_request = LoRARequest(
99
+ lora_name if lora_name else str(lora_id), lora_id, self.config.lora_path
100
+ )
101
+ outputs = self.llm.generate(inputs, sampling_params, lora_request=lora_request)
102
+ generated_texts = [output.outputs[0].text.strip() for output in outputs]
103
+ # Validate outputs
104
+ for generated, expected in zip(generated_texts, expected_outputs):
105
+ assert expected.startswith(generated), (
106
+ f"Generated text {generated} doesn't match expected pattern {expected}"
107
+ )
108
+
109
+ def run_beam_search_test(
110
+ self,
111
+ images: list[ImageAsset],
112
+ expected_outputs: list[list[str]],
113
+ lora_id: int | None = None,
114
+ temperature: float = 0,
115
+ beam_width: int = 2,
116
+ max_tokens: int = 5,
117
+ ):
118
+ beam_search_params = BeamSearchParams(
119
+ beam_width=beam_width, max_tokens=max_tokens, temperature=temperature
120
+ )
121
+
122
+ inputs = [
123
+ {
124
+ "prompt": self.PROMPT_TEMPLATE,
125
+ "multi_modal_data": {"image": asset.pil_image},
126
+ }
127
+ for asset in images
128
+ ]
129
+
130
+ lora_request = LoRARequest(str(lora_id), lora_id, self.config.lora_path)
131
+ outputs = self.llm.beam_search(
132
+ inputs, beam_search_params, lora_request=lora_request
133
+ )
134
+
135
+ for output_obj, expected_texts in zip(outputs, expected_outputs):
136
+ output_texts = [seq.text for seq in output_obj.sequences]
137
+
138
+ for output_text, expected_text in zip(output_texts, expected_texts):
139
+ # NOTE beam search .text contains the whole text including inputs
140
+ assert output_text.endswith(expected_text), (
141
+ f"Generated {output_text} does not match expected {expected_text}"
142
+ )
143
+
144
+
145
+ TEST_IMAGES = [
146
+ ImageAsset("stop_sign"),
147
+ ImageAsset("cherry_blossom"),
148
+ ]
149
+
150
+ EXPECTED_OUTPUTS = [
151
+ "A red stop sign stands prominently in the foreground, with a traditional Chinese gate and a black SUV in the background, illustrating a blend of modern and cultural elements.", # noqa: E501
152
+ "A majestic skyscraper stands tall, partially obscured by a vibrant canopy of cherry blossoms, against a clear blue sky.", # noqa: E501
153
+ ]
154
+
155
+ EXPECTED_OUTPUTS_LANGUAGE = [
156
+ "A stop sign is shown in an Asian city, with buildings and a car in the "
157
+ "background.",
158
+ "The Tokyo Skytree can be seen behind the pink blossoms of the cherry trees.",
159
+ ]
160
+
161
+ EXPECTED_OUTPUTS_VISION = [
162
+ "A stop sign in front of oriental buildings.",
163
+ "A tree with pink flowers in front of it and a blue sky behind the flowers.",
164
+ ]
165
+
166
+ EXPECTED_OUTPUTS_VISION_NO_CONNECTOR = [
167
+ "A stop sign is located on the street of a Chinese neighborhood.",
168
+ "A closeup shot of the Tokyo Skytree with pink flowers in the foreground.",
169
+ ]
170
+
171
+ EXPECTED_BEAM_SEARCH_OUTPUTS = [
172
+ [
173
+ "A majestic skyscraper stands",
174
+ "A majestic tower stands tall",
175
+ ],
176
+ ]
177
+
178
+ QWEN2VL_MODEL_PATH = "Qwen/Qwen2-VL-2B-Instruct"
179
+ QWEN25VL_MODEL_PATH = "Qwen/Qwen2.5-VL-3B-Instruct"
180
+ QWEN3VL_MODEL_PATH = "Qwen/Qwen3-VL-4B-Instruct"
181
+
182
+
183
+ def test_qwen2vl_lora(qwen2vl_lora_files):
184
+ """Test Qwen 2.0 VL model with LoRA"""
185
+ config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files)
186
+ tester = Qwen2VLTester(config)
187
+
188
+ # Test with different LoRA IDs
189
+ for lora_id in [1, 2]:
190
+ tester.run_test(TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id)
191
+
192
+
193
+ def test_qwen2vl_lora_beam_search(qwen2vl_lora_files):
194
+ """Test Qwen 2.0 VL model with LoRA through beam search."""
195
+ config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files)
196
+ tester = Qwen2VLTester(config)
197
+
198
+ # Test with different LoRA IDs
199
+ for lora_id in [1, 2]:
200
+ # NOTE currently, we only test cherry blossom since stop sign
201
+ # output is slightly different for v1; - the root cause is likely
202
+ # independent of the intent of this test, which is to ensure beam
203
+ # search passes through lora through correctly.
204
+ tester.run_beam_search_test(
205
+ [ImageAsset("cherry_blossom")],
206
+ expected_outputs=EXPECTED_BEAM_SEARCH_OUTPUTS,
207
+ lora_id=lora_id,
208
+ )
209
+
210
+
211
+ @pytest.mark.skipif(
212
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
213
+ )
214
+ def test_qwen25vl_lora(qwen25vl_lora_files):
215
+ """Test Qwen 2.5 VL model with LoRA"""
216
+ config = TestConfig(model_path=QWEN25VL_MODEL_PATH, lora_path=qwen25vl_lora_files)
217
+ tester = Qwen2VLTester(config)
218
+
219
+ # Test with different LoRA IDs
220
+ for lora_id in [1, 2]:
221
+ tester.run_test(TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id)
222
+
223
+
224
+ @pytest.mark.skipif(
225
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
226
+ )
227
+ def test_qwen25vl_vision_lora(qwen25vl_vision_lora_files):
228
+ config = TestConfig(
229
+ model_path=QWEN25VL_MODEL_PATH,
230
+ lora_path=qwen25vl_vision_lora_files,
231
+ # Currently, tower_connector_lora is incompatible with
232
+ # the multi-modal processor cache.
233
+ # TODO: Remove this restriction
234
+ mm_processor_cache_gb=0,
235
+ enable_tower_connector_lora=True,
236
+ )
237
+ tester = Qwen2VLTester(config)
238
+ for lora_id in [1, 2]:
239
+ tester.run_test(
240
+ TEST_IMAGES,
241
+ expected_outputs=EXPECTED_OUTPUTS,
242
+ lora_id=lora_id,
243
+ )
244
+
245
+
246
+ def test_qwen3vl_vision_lora(qwen3vl_vision_lora_files):
247
+ config = TestConfig(
248
+ model_path=QWEN3VL_MODEL_PATH,
249
+ lora_path=qwen3vl_vision_lora_files,
250
+ # Currently, tower_connector_lora is incompatible with
251
+ # the multi-modal processor cache.
252
+ # TODO: Remove this restriction
253
+ mm_processor_cache_gb=0,
254
+ enable_tower_connector_lora=True,
255
+ )
256
+ tester = Qwen2VLTester(config)
257
+ for lora_id in [1, 2]:
258
+ tester.run_test(
259
+ TEST_IMAGES,
260
+ expected_outputs=EXPECTED_OUTPUTS,
261
+ lora_id=lora_id,
262
+ )
263
+
264
+
265
+ def test_qwen2vl_multiple_lora_types(
266
+ qwen2vl_language_lora_files,
267
+ qwen2vl_vision_tower_connector_lora_files,
268
+ qwen2vl_vision_tower_lora_files,
269
+ ):
270
+ """
271
+ Test multiple LoRA adapter types (language, vision tower + connector,
272
+ vision tower only) using the same LLM instance to verify mm_encoder_cache
273
+ behavior with different LoRA requests.
274
+
275
+ By reusing the same LLM instance across different LoRA requests, we ensure that
276
+ the multimodal encoder cache correctly manages state transitions between
277
+ language-only and vision-enabled LoRA adapters.
278
+ """
279
+ config = TestConfig(
280
+ model_path=QWEN2VL_MODEL_PATH,
281
+ # We'll override the lora_path for each specific test, but need to provide
282
+ # an initial path for initialization
283
+ lora_path=qwen2vl_language_lora_files,
284
+ # Currently, tower_connector_lora is incompatible with
285
+ # the multi-modal processor cache.
286
+ # TODO: Remove this restriction
287
+ mm_processor_cache_gb=0,
288
+ enable_tower_connector_lora=True,
289
+ )
290
+ tester = Qwen2VLTester(config)
291
+
292
+ # Test 1: Language-only LoRA adapter
293
+ tester.config.lora_path = qwen2vl_language_lora_files
294
+ for lora_id in [1, 2]:
295
+ tester.run_test(
296
+ TEST_IMAGES,
297
+ expected_outputs=EXPECTED_OUTPUTS_LANGUAGE,
298
+ lora_id=lora_id,
299
+ lora_name="language_only",
300
+ )
301
+
302
+ # Test 2: Vision tower + connector LoRA adapter
303
+ tester.config.lora_path = qwen2vl_vision_tower_connector_lora_files
304
+ for lora_id in [3, 4]:
305
+ tester.run_test(
306
+ TEST_IMAGES,
307
+ expected_outputs=EXPECTED_OUTPUTS_VISION,
308
+ lora_id=lora_id,
309
+ lora_name="vision_tower_connector",
310
+ )
311
+
312
+ # Test 3: Vision tower only LoRA adapter (no connector)
313
+ tester.config.lora_path = qwen2vl_vision_tower_lora_files
314
+ for lora_id in [5, 6]:
315
+ tester.run_test(
316
+ TEST_IMAGES,
317
+ expected_outputs=EXPECTED_OUTPUTS_VISION_NO_CONNECTOR,
318
+ lora_id=lora_id,
319
+ lora_name="vision_tower",
320
+ )
asian_ai/performance/vLLM/tests/lora/test_resolver.py ADDED
@@ -0,0 +1,75 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+
5
+ import pytest
6
+
7
+ from vllm.lora.request import LoRARequest
8
+ from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
9
+
10
+
11
+ class DummyLoRAResolver(LoRAResolver):
12
+ """A dummy LoRA resolver for testing."""
13
+
14
+ async def resolve_lora(
15
+ self, base_model_name: str, lora_name: str
16
+ ) -> LoRARequest | None:
17
+ if lora_name == "test_lora":
18
+ return LoRARequest(
19
+ lora_name=lora_name,
20
+ lora_path=f"/dummy/path/{base_model_name}/{lora_name}",
21
+ lora_int_id=abs(hash(lora_name)),
22
+ )
23
+ return None
24
+
25
+
26
+ def test_resolver_registry_registration():
27
+ """Test basic resolver registration functionality."""
28
+ registry = LoRAResolverRegistry
29
+ resolver = DummyLoRAResolver()
30
+
31
+ # Register a new resolver
32
+ registry.register_resolver("dummy", resolver)
33
+ assert "dummy" in registry.get_supported_resolvers()
34
+
35
+ # Get registered resolver
36
+ retrieved_resolver = registry.get_resolver("dummy")
37
+ assert retrieved_resolver is resolver
38
+
39
+
40
+ def test_resolver_registry_duplicate_registration():
41
+ """Test registering a resolver with an existing name."""
42
+ registry = LoRAResolverRegistry
43
+ resolver1 = DummyLoRAResolver()
44
+ resolver2 = DummyLoRAResolver()
45
+
46
+ registry.register_resolver("dummy", resolver1)
47
+ registry.register_resolver("dummy", resolver2)
48
+
49
+ assert registry.get_resolver("dummy") is resolver2
50
+
51
+
52
+ def test_resolver_registry_unknown_resolver():
53
+ """Test getting a non-existent resolver."""
54
+ registry = LoRAResolverRegistry
55
+
56
+ with pytest.raises(KeyError, match="not found"):
57
+ registry.get_resolver("unknown_resolver")
58
+
59
+
60
+ @pytest.mark.asyncio
61
+ async def test_dummy_resolver_resolve():
62
+ """Test the dummy resolver's resolve functionality."""
63
+ dummy_resolver = DummyLoRAResolver()
64
+ base_model_name = "base_model_test"
65
+ lora_name = "test_lora"
66
+
67
+ # Test successful resolution
68
+ result = await dummy_resolver.resolve_lora(base_model_name, lora_name)
69
+ assert isinstance(result, LoRARequest)
70
+ assert result.lora_name == lora_name
71
+ assert result.lora_path == f"/dummy/path/{base_model_name}/{lora_name}"
72
+
73
+ # Test failed resolution
74
+ result = await dummy_resolver.resolve_lora(base_model_name, "nonexistent_lora")
75
+ assert result is None
asian_ai/performance/vLLM/tests/lora/test_transformers_model.py ADDED
@@ -0,0 +1,116 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import pytest
5
+
6
+ import vllm
7
+ from vllm.lora.request import LoRARequest
8
+ from vllm.platforms import current_platform
9
+
10
+ from ..utils import create_new_process_for_each_test, multi_gpu_test
11
+
12
+ MODEL_PATH = "hmellor/Ilama-3.2-1B"
13
+
14
+ PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
15
+
16
+ EXPECTED_LORA_OUTPUT = [
17
+ "SELECT count(*) FROM singer",
18
+ "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'", # noqa: E501
19
+ "SELECT DISTINCT Country FROM singer WHERE Age > 20",
20
+ ]
21
+
22
+
23
+ def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
24
+ prompts = [
25
+ PROMPT_TEMPLATE.format(query="How many singers do we have?"),
26
+ PROMPT_TEMPLATE.format(
27
+ query="What is the average, minimum, and maximum age of all singers from France?" # noqa: E501
28
+ ),
29
+ PROMPT_TEMPLATE.format(
30
+ query="What are all distinct countries where singers above age 20 are from?" # noqa: E501
31
+ ),
32
+ ]
33
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=32)
34
+ outputs = llm.generate(
35
+ prompts,
36
+ sampling_params,
37
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
38
+ )
39
+ # Print the outputs.
40
+ generated_texts: list[str] = []
41
+ for output in outputs:
42
+ prompt = output.prompt
43
+ generated_text = output.outputs[0].text.strip()
44
+ generated_texts.append(generated_text)
45
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
46
+ return generated_texts
47
+
48
+
49
+ def test_ilama_lora(ilama_lora_files):
50
+ llm = vllm.LLM(
51
+ MODEL_PATH,
52
+ max_model_len=1024,
53
+ enable_lora=True,
54
+ max_loras=4,
55
+ max_lora_rank=16,
56
+ trust_remote_code=True,
57
+ enable_chunked_prefill=True,
58
+ )
59
+
60
+ output1 = do_sample(llm, ilama_lora_files, lora_id=1)
61
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
62
+ assert output1[i] == EXPECTED_LORA_OUTPUT[i]
63
+ output2 = do_sample(llm, ilama_lora_files, lora_id=2)
64
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
65
+ assert output2[i] == EXPECTED_LORA_OUTPUT[i]
66
+
67
+
68
+ @pytest.mark.skipif(
69
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
70
+ )
71
+ @multi_gpu_test(num_gpus=4)
72
+ @create_new_process_for_each_test()
73
+ def test_ilama_lora_tp4(ilama_lora_files):
74
+ llm = vllm.LLM(
75
+ MODEL_PATH,
76
+ max_model_len=1024,
77
+ enable_lora=True,
78
+ max_loras=4,
79
+ max_lora_rank=16,
80
+ tensor_parallel_size=4,
81
+ trust_remote_code=True,
82
+ fully_sharded_loras=False,
83
+ enable_chunked_prefill=True,
84
+ )
85
+
86
+ output1 = do_sample(llm, ilama_lora_files, lora_id=1)
87
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
88
+ assert output1[i] == EXPECTED_LORA_OUTPUT[i]
89
+ output2 = do_sample(llm, ilama_lora_files, lora_id=2)
90
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
91
+ assert output2[i] == EXPECTED_LORA_OUTPUT[i]
92
+
93
+
94
+ @pytest.mark.skipif(
95
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
96
+ )
97
+ @multi_gpu_test(num_gpus=4)
98
+ @create_new_process_for_each_test()
99
+ def test_ilama_lora_tp4_fully_sharded_loras(ilama_lora_files):
100
+ llm = vllm.LLM(
101
+ MODEL_PATH,
102
+ max_model_len=1024,
103
+ enable_lora=True,
104
+ max_loras=4,
105
+ max_lora_rank=16,
106
+ tensor_parallel_size=4,
107
+ trust_remote_code=True,
108
+ fully_sharded_loras=True,
109
+ enable_chunked_prefill=True,
110
+ )
111
+ output1 = do_sample(llm, ilama_lora_files, lora_id=1)
112
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
113
+ assert output1[i] == EXPECTED_LORA_OUTPUT[i]
114
+ output2 = do_sample(llm, ilama_lora_files, lora_id=2)
115
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
116
+ assert output2[i] == EXPECTED_LORA_OUTPUT[i]
asian_ai/performance/vLLM/tests/lora/test_utils.py ADDED
@@ -0,0 +1,201 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ from collections import OrderedDict
5
+ from typing import NamedTuple
6
+ from unittest.mock import MagicMock, patch
7
+
8
+ import pytest
9
+ from huggingface_hub.utils import HfHubHTTPError
10
+ from torch import nn
11
+
12
+ from vllm.lora.utils import (
13
+ get_adapter_absolute_path,
14
+ parse_fine_tuned_lora_name,
15
+ replace_submodule,
16
+ )
17
+ from vllm.model_executor.models.utils import WeightsMapper
18
+
19
+
20
+ class LoRANameParserTestConfig(NamedTuple):
21
+ name: str
22
+ module_name: str
23
+ is_lora_a: bool
24
+ weights_mapper: WeightsMapper | None = None
25
+
26
+
27
+ def test_parse_fine_tuned_lora_name_valid():
28
+ fixture = [
29
+ LoRANameParserTestConfig(
30
+ "base_model.model.lm_head.lora_A.weight", "lm_head", True, False
31
+ ),
32
+ LoRANameParserTestConfig(
33
+ "base_model.model.lm_head.lora_B.weight", "lm_head", False, False
34
+ ),
35
+ LoRANameParserTestConfig(
36
+ "base_model.model.model.embed_tokens.lora_embedding_A",
37
+ "model.embed_tokens",
38
+ True,
39
+ ),
40
+ LoRANameParserTestConfig(
41
+ "base_model.model.model.embed_tokens.lora_embedding_B",
42
+ "model.embed_tokens",
43
+ False,
44
+ ),
45
+ LoRANameParserTestConfig(
46
+ "base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
47
+ "model.layers.9.mlp.down_proj",
48
+ True,
49
+ ),
50
+ LoRANameParserTestConfig(
51
+ "base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
52
+ "model.layers.9.mlp.down_proj",
53
+ False,
54
+ ),
55
+ LoRANameParserTestConfig(
56
+ "language_model.layers.9.mlp.down_proj.lora_A.weight",
57
+ "language_model.layers.9.mlp.down_proj",
58
+ True,
59
+ ),
60
+ LoRANameParserTestConfig(
61
+ "language_model.layers.9.mlp.down_proj.lora_B.weight",
62
+ "language_model.layers.9.mlp.down_proj",
63
+ False,
64
+ ),
65
+ # Test with WeightsMapper
66
+ LoRANameParserTestConfig(
67
+ "base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
68
+ "language_model.model.layers.9.mlp.down_proj",
69
+ True,
70
+ weights_mapper=WeightsMapper(
71
+ orig_to_new_prefix={"model.": "language_model.model."}
72
+ ),
73
+ ),
74
+ LoRANameParserTestConfig(
75
+ "base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
76
+ "language_model.model.layers.9.mlp.down_proj",
77
+ False,
78
+ weights_mapper=WeightsMapper(
79
+ orig_to_new_prefix={"model.": "language_model.model."}
80
+ ),
81
+ ),
82
+ LoRANameParserTestConfig(
83
+ "model.layers.9.mlp.down_proj.lora_A.weight",
84
+ "language_model.model.layers.9.mlp.down_proj",
85
+ True,
86
+ weights_mapper=WeightsMapper(
87
+ orig_to_new_prefix={"model.": "language_model.model."}
88
+ ),
89
+ ),
90
+ LoRANameParserTestConfig(
91
+ "model.layers.9.mlp.down_proj.lora_B.weight",
92
+ "language_model.model.layers.9.mlp.down_proj",
93
+ False,
94
+ weights_mapper=WeightsMapper(
95
+ orig_to_new_prefix={"model.": "language_model.model."}
96
+ ),
97
+ ),
98
+ ]
99
+ for name, module_name, is_lora_a, weights_mapper in fixture:
100
+ assert (module_name, is_lora_a) == parse_fine_tuned_lora_name(
101
+ name, weights_mapper
102
+ )
103
+
104
+
105
+ def test_parse_fine_tuned_lora_name_invalid():
106
+ fixture = {
107
+ "base_model.weight",
108
+ "base_model.model.weight",
109
+ }
110
+ for name in fixture:
111
+ with pytest.raises(ValueError, match="unsupported LoRA weight"):
112
+ parse_fine_tuned_lora_name(name)
113
+
114
+
115
+ def test_replace_submodule():
116
+ model = nn.Sequential(
117
+ OrderedDict(
118
+ [
119
+ ("dense1", nn.Linear(764, 100)),
120
+ ("act1", nn.ReLU()),
121
+ ("dense2", nn.Linear(100, 50)),
122
+ (
123
+ "seq1",
124
+ nn.Sequential(
125
+ OrderedDict(
126
+ [
127
+ ("dense1", nn.Linear(100, 10)),
128
+ ("dense2", nn.Linear(10, 50)),
129
+ ]
130
+ )
131
+ ),
132
+ ),
133
+ ("act2", nn.ReLU()),
134
+ ("output", nn.Linear(50, 10)),
135
+ ("outact", nn.Sigmoid()),
136
+ ]
137
+ )
138
+ )
139
+
140
+ sigmoid = nn.Sigmoid()
141
+
142
+ replace_submodule(model, "act1", sigmoid)
143
+ assert dict(model.named_modules())["act1"] == sigmoid
144
+
145
+ dense2 = nn.Linear(1, 5)
146
+ replace_submodule(model, "seq1.dense2", dense2)
147
+ assert dict(model.named_modules())["seq1.dense2"] == dense2
148
+
149
+
150
+ # Unit tests for get_adapter_absolute_path
151
+ @patch("os.path.isabs")
152
+ def test_get_adapter_absolute_path_absolute(mock_isabs):
153
+ path = "/absolute/path/to/lora"
154
+ mock_isabs.return_value = True
155
+ assert get_adapter_absolute_path(path) == path
156
+
157
+
158
+ @patch("os.path.expanduser")
159
+ def test_get_adapter_absolute_path_expanduser(mock_expanduser):
160
+ # Path with ~ that needs to be expanded
161
+ path = "~/relative/path/to/lora"
162
+ absolute_path = "/home/user/relative/path/to/lora"
163
+ mock_expanduser.return_value = absolute_path
164
+ assert get_adapter_absolute_path(path) == absolute_path
165
+
166
+
167
+ @patch("os.path.exists")
168
+ @patch("os.path.abspath")
169
+ def test_get_adapter_absolute_path_local_existing(mock_abspath, mock_exist):
170
+ # Relative path that exists locally
171
+ path = "relative/path/to/lora"
172
+ absolute_path = "/absolute/path/to/lora"
173
+ mock_exist.return_value = True
174
+ mock_abspath.return_value = absolute_path
175
+ assert get_adapter_absolute_path(path) == absolute_path
176
+
177
+
178
+ @patch("huggingface_hub.HfApi.snapshot_download")
179
+ @patch("os.path.exists")
180
+ def test_get_adapter_absolute_path_huggingface(mock_exist, mock_snapshot_download):
181
+ # Hugging Face model identifier
182
+ path = "org/repo"
183
+ absolute_path = "/mock/snapshot/path"
184
+ mock_exist.return_value = False
185
+ mock_snapshot_download.return_value = absolute_path
186
+ assert get_adapter_absolute_path(path) == absolute_path
187
+
188
+
189
+ @patch("huggingface_hub.HfApi.snapshot_download")
190
+ @patch("os.path.exists")
191
+ def test_get_adapter_absolute_path_huggingface_error(
192
+ mock_exist, mock_snapshot_download
193
+ ):
194
+ # Hugging Face model identifier with download error
195
+ path = "org/repo"
196
+ mock_exist.return_value = False
197
+ mock_snapshot_download.side_effect = HfHubHTTPError(
198
+ "failed to query model info",
199
+ response=MagicMock(),
200
+ )
201
+ assert get_adapter_absolute_path(path) == path
asian_ai/performance/vLLM/tests/lora/test_whisper.py ADDED
@@ -0,0 +1,134 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ """
4
+ Integration tests for Whisper models with LoRA adapters.
5
+
6
+ These tests verify that Whisper models can correctly load and use LoRA adapters
7
+ for speech-to-text transcription tasks.
8
+ """
9
+
10
+ import pytest
11
+
12
+ import vllm
13
+ from vllm.assets.audio import AudioAsset
14
+ from vllm.lora.request import LoRARequest
15
+ from vllm.platforms import current_platform
16
+
17
+ from ..utils import create_new_process_for_each_test
18
+
19
+ # Model configuration
20
+ WHISPER_MODEL = "openai/whisper-small"
21
+
22
+ # Test prompts for Whisper transcription
23
+ WHISPER_PROMPT = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
24
+
25
+ # Note: whisper_lora_files fixture is defined in conftest.py
26
+
27
+
28
+ @pytest.fixture(autouse=True)
29
+ def use_spawn_for_whisper(monkeypatch):
30
+ """Whisper has issues with forked workers, use spawn instead."""
31
+ monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
32
+
33
+
34
+ def create_whisper_llm(
35
+ enable_lora: bool = True, max_loras: int = 2, attn_backend: str | None = None
36
+ ):
37
+ """Create a Whisper LLM instance with optional LoRA support."""
38
+ return vllm.LLM(
39
+ model=WHISPER_MODEL,
40
+ enable_lora=enable_lora,
41
+ max_loras=max_loras if enable_lora else 1,
42
+ max_lora_rank=64,
43
+ max_model_len=448,
44
+ dtype="half",
45
+ enforce_eager=True, # For stability in tests
46
+ attention_config={"backend": attn_backend},
47
+ )
48
+
49
+
50
+ def run_whisper_inference(
51
+ llm: vllm.LLM,
52
+ lora_path: str | None = None,
53
+ lora_id: int = 1,
54
+ ) -> list[str]:
55
+ """Run Whisper inference with optional LoRA adapter."""
56
+ # Load test audio
57
+ audio_asset = AudioAsset("mary_had_lamb")
58
+ audio_data = audio_asset.audio_and_sample_rate
59
+
60
+ inputs = [
61
+ {
62
+ "prompt": WHISPER_PROMPT,
63
+ "multi_modal_data": {"audio": audio_data},
64
+ }
65
+ ]
66
+
67
+ sampling_params = vllm.SamplingParams(
68
+ temperature=0,
69
+ max_tokens=200,
70
+ )
71
+
72
+ # Prepare LoRA request if adapter path is provided
73
+ lora_request = None
74
+ if lora_path:
75
+ lora_request = LoRARequest(
76
+ lora_name=f"whisper_lora_{lora_id}",
77
+ lora_int_id=lora_id,
78
+ lora_path=lora_path,
79
+ )
80
+
81
+ outputs = llm.generate(inputs, sampling_params, lora_request=lora_request)
82
+
83
+ return [output.outputs[0].text for output in outputs]
84
+
85
+
86
+ @create_new_process_for_each_test()
87
+ def test_whisper_lora_inference(whisper_lora_files):
88
+ """Test basic Whisper inference with a LoRA adapter.
89
+
90
+ This test verifies that:
91
+ 1. Whisper model can be loaded with LoRA support enabled
92
+ 2. A LoRA adapter can be applied during inference
93
+ 3. The model produces valid transcription output
94
+ """
95
+ llm = create_whisper_llm(enable_lora=True)
96
+
97
+ # Run inference with LoRA
98
+ outputs = run_whisper_inference(llm, lora_path=whisper_lora_files, lora_id=1)
99
+
100
+ # Verify we got a non-empty transcription
101
+ assert len(outputs) == 1
102
+ assert len(outputs[0]) > 0, "Expected non-empty transcription output"
103
+
104
+ # The output should contain some recognizable words from the audio
105
+ # (Mary had a little lamb)
106
+ print(f"Transcription output: {outputs[0]}")
107
+
108
+
109
+ @create_new_process_for_each_test()
110
+ def test_whisper_multi_lora(whisper_lora_files):
111
+ """Test Whisper with multiple LoRA adapter IDs.
112
+
113
+ This test verifies that the same LoRA adapter can be loaded with
114
+ different IDs and produce consistent results.
115
+ """
116
+ llm = create_whisper_llm(
117
+ enable_lora=True,
118
+ max_loras=4,
119
+ attn_backend="TRITON_ATTN" if current_platform.is_rocm() else None,
120
+ )
121
+
122
+ # Test with different LoRA IDs using the same adapter
123
+ outputs_lora1 = run_whisper_inference(llm, lora_path=whisper_lora_files, lora_id=1)
124
+ outputs_lora2 = run_whisper_inference(llm, lora_path=whisper_lora_files, lora_id=2)
125
+
126
+ # Both should produce valid outputs
127
+ assert len(outputs_lora1[0]) > 0
128
+ assert len(outputs_lora2[0]) > 0
129
+
130
+ # Same adapter with different IDs should produce same output
131
+ assert outputs_lora1 == outputs_lora2, (
132
+ f"Expected same outputs for same adapter with different IDs. "
133
+ f"Got: {outputs_lora1} vs {outputs_lora2}"
134
+ )
asian_ai/performance/vLLM/tests/lora/test_worker.py ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import os
5
+ import random
6
+ import tempfile
7
+ from unittest.mock import patch
8
+
9
+ from vllm.config import (
10
+ CacheConfig,
11
+ DeviceConfig,
12
+ ModelConfig,
13
+ ParallelConfig,
14
+ SchedulerConfig,
15
+ VllmConfig,
16
+ set_current_vllm_config,
17
+ )
18
+ from vllm.config.load import LoadConfig
19
+ from vllm.config.lora import LoRAConfig
20
+ from vllm.lora.model_manager import LoRAMapping
21
+ from vllm.lora.request import LoRARequest
22
+ from vllm.platforms import current_platform
23
+
24
+ if current_platform.is_xpu():
25
+ from vllm.v1.worker.xpu_worker import XPUWorker as Worker
26
+ else:
27
+ from vllm.v1.worker.gpu_worker import Worker
28
+
29
+ MODEL_PATH = "Qwen/Qwen3-0.6B"
30
+ NUM_LORAS = 16
31
+
32
+ DEVICE_TYPE = current_platform.device_type
33
+
34
+
35
+ @patch.dict(os.environ, {"RANK": "0"})
36
+ def test_worker_apply_lora(qwen3_lora_files):
37
+ def set_active_loras(worker: Worker, lora_requests: list[LoRARequest]):
38
+ lora_mapping = LoRAMapping([], [])
39
+
40
+ worker.model_runner.lora_manager.set_active_adapters(
41
+ lora_requests, lora_mapping
42
+ )
43
+
44
+ model_config = ModelConfig(
45
+ MODEL_PATH,
46
+ seed=0,
47
+ dtype="float16",
48
+ max_model_len=127,
49
+ enforce_eager=True,
50
+ )
51
+
52
+ vllm_config = VllmConfig(
53
+ model_config=model_config,
54
+ load_config=LoadConfig(
55
+ download_dir=None,
56
+ load_format="dummy",
57
+ ),
58
+ parallel_config=ParallelConfig(
59
+ pipeline_parallel_size=1,
60
+ tensor_parallel_size=1,
61
+ data_parallel_size=1,
62
+ ),
63
+ scheduler_config=SchedulerConfig(
64
+ max_model_len=model_config.max_model_len,
65
+ is_encoder_decoder=model_config.is_encoder_decoder,
66
+ runner_type="generate",
67
+ max_num_batched_tokens=32,
68
+ max_num_seqs=32,
69
+ max_num_partial_prefills=32,
70
+ ),
71
+ device_config=DeviceConfig(DEVICE_TYPE),
72
+ cache_config=CacheConfig(
73
+ block_size=16,
74
+ cache_dtype="auto",
75
+ ),
76
+ lora_config=LoRAConfig(
77
+ max_lora_rank=8, max_cpu_loras=NUM_LORAS, max_loras=NUM_LORAS
78
+ ),
79
+ )
80
+ worker = Worker(
81
+ vllm_config=vllm_config,
82
+ local_rank=0,
83
+ rank=0,
84
+ distributed_init_method=f"file://{tempfile.mkstemp()[1]}",
85
+ )
86
+
87
+ with set_current_vllm_config(vllm_config):
88
+ worker.init_device()
89
+ worker.load_model()
90
+
91
+ set_active_loras(worker, [])
92
+ assert worker.list_loras() == set()
93
+
94
+ lora_requests = [
95
+ LoRARequest(str(i + 1), i + 1, qwen3_lora_files) for i in range(NUM_LORAS)
96
+ ]
97
+
98
+ set_active_loras(worker, lora_requests)
99
+ assert worker.list_loras() == {
100
+ lora_request.lora_int_id for lora_request in lora_requests
101
+ }
102
+
103
+ for i in range(NUM_LORAS):
104
+ random.seed(i)
105
+ iter_lora_requests = random.choices(
106
+ lora_requests, k=random.randint(1, NUM_LORAS)
107
+ )
108
+ random.shuffle(iter_lora_requests)
109
+ iter_lora_requests = iter_lora_requests[: -random.randint(0, NUM_LORAS)]
110
+ set_active_loras(worker, lora_requests)
111
+ assert worker.list_loras().issuperset(
112
+ {lora_request.lora_int_id for lora_request in iter_lora_requests}
113
+ )
asian_ai/performance/vLLM/tests/lora/utils.py ADDED
@@ -0,0 +1,410 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import json
5
+ import os
6
+ from dataclasses import dataclass
7
+
8
+ import torch
9
+ from safetensors.torch import save_file
10
+
11
+ from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
12
+ from vllm.platforms import current_platform
13
+
14
+ DEVICE_TYPE = current_platform.device_type
15
+
16
+
17
+ class DummyLoRAManager:
18
+ def __init__(self, device: torch.device = f"{DEVICE_TYPE}:0"):
19
+ super().__init__()
20
+ self._loras: dict[str, LoRALayerWeights] = {}
21
+ self._device = device
22
+
23
+ def set_module_lora(self, module_name: str, lora: LoRALayerWeights):
24
+ self._loras[module_name] = lora
25
+
26
+ def get_module_lora(self, module_name: str) -> LoRALayerWeights:
27
+ return self._loras[module_name]
28
+
29
+ def init_random_lora(
30
+ self,
31
+ module_name: str,
32
+ weight: torch.Tensor,
33
+ rank: int = 8,
34
+ ):
35
+ lora = LoRALayerWeights(
36
+ module_name,
37
+ rank=rank,
38
+ lora_alpha=1,
39
+ lora_a=torch.rand(
40
+ [rank, weight.shape[1]], dtype=weight.dtype, device=self._device
41
+ ),
42
+ lora_b=torch.rand(
43
+ [weight.shape[0], rank], dtype=weight.dtype, device=self._device
44
+ ),
45
+ )
46
+ self.set_module_lora(module_name, lora)
47
+
48
+ return lora
49
+
50
+ def init_lora(
51
+ self,
52
+ module_name: str,
53
+ input_dim: int,
54
+ output_dim: int,
55
+ rank=8,
56
+ noop=False,
57
+ embeddings_tensor=None,
58
+ ):
59
+ lora = LoRALayerWeights(
60
+ module_name,
61
+ rank=rank,
62
+ lora_alpha=1,
63
+ lora_a=torch.rand([rank, input_dim], device=DEVICE_TYPE),
64
+ lora_b=torch.rand([output_dim, input_dim], device=DEVICE_TYPE),
65
+ embeddings_tensor=embeddings_tensor,
66
+ )
67
+ self.set_module_lora(module_name, lora)
68
+ return lora
69
+
70
+ def reset_lora(self):
71
+ self._loras = {}
72
+
73
+ def init_packed_lora(
74
+ self,
75
+ module_name: str,
76
+ input_dim: int,
77
+ output_dims: list[int],
78
+ noop_lora_index: list[int] | None = None,
79
+ rank: int = 8,
80
+ ):
81
+ base_loras: list[LoRALayerWeights] = []
82
+ noop_lora_index_set = set(noop_lora_index or [])
83
+
84
+ for i, out_dim in enumerate(output_dims):
85
+ base_lora = self.init_lora(
86
+ module_name + "_000_" + str(i),
87
+ input_dim,
88
+ out_dim,
89
+ rank=rank,
90
+ noop=i in noop_lora_index_set,
91
+ )
92
+ base_loras.append(base_lora)
93
+ packed_lora = PackedLoRALayerWeights.pack(base_loras)
94
+ self.set_module_lora(module_name, packed_lora)
95
+ return packed_lora
96
+
97
+
98
+ def assert_close(a, b):
99
+ rtol, atol = {
100
+ torch.float16: (6e-2, 6e-2),
101
+ torch.bfloat16: (6e-2, 6e-2),
102
+ torch.float32: (1e-2, 1e-2),
103
+ }[a.dtype]
104
+ torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
105
+
106
+
107
+ @dataclass
108
+ class PunicaTensors:
109
+ inputs_tensor: torch.Tensor
110
+ lora_weights: torch.Tensor | list[torch.Tensor]
111
+ our_out_tensor: torch.Tensor
112
+ ref_out_tensor: torch.Tensor
113
+ b_seq_start_loc: torch.Tensor
114
+ prompt_lora_mapping: torch.Tensor
115
+ seq_len_tensor: torch.Tensor
116
+ token_lora_mapping: torch.Tensor
117
+
118
+ def meta(self) -> tuple[int, int]:
119
+ """
120
+ Infer max_seq_length and token_nums from the tensors
121
+ and return them.
122
+ """
123
+ max_seq_length = self.seq_len_tensor.max()
124
+ token_nums = self.seq_len_tensor.sum().item()
125
+ if isinstance(max_seq_length, tuple):
126
+ max_seq_length = max_seq_length[0].item()
127
+ else:
128
+ max_seq_length = max_seq_length.item()
129
+ return max_seq_length, token_nums
130
+
131
+
132
+ def generate_data(
133
+ batches,
134
+ hidden_size,
135
+ lora_nums,
136
+ max_rank,
137
+ seq_length,
138
+ dtype,
139
+ op_type,
140
+ device,
141
+ ) -> PunicaTensors:
142
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1, (batches,)).to(device)
143
+ b_seq_start_loc = torch.cumsum(
144
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
145
+ dim=0,
146
+ ).to(device)
147
+ total_tokens = seq_len_tensor.sum()
148
+ if op_type == "shrink":
149
+ inputs_tensor = torch.rand((total_tokens, hidden_size), dtype=dtype).to(device)
150
+ lora_weights = torch.rand(
151
+ (lora_nums, max_rank, hidden_size), # col-major
152
+ dtype=dtype,
153
+ ).to(device)
154
+ # shrink op need atomic_add, so output is initinized by 0
155
+ ref_out_tensor = torch.zeros(
156
+ (total_tokens, max_rank), dtype=dtype, device=inputs_tensor.device
157
+ )
158
+ # NOTE shrink kernel using torch.float32 as output type
159
+ our_out_tensor = torch.zeros((total_tokens, max_rank), dtype=torch.float32).to(
160
+ device
161
+ )
162
+ else:
163
+ inputs_tensor = torch.rand(
164
+ (total_tokens, max_rank),
165
+ dtype=dtype,
166
+ ).to(device)
167
+ lora_weights = torch.rand(
168
+ (lora_nums, hidden_size, max_rank), # col-major
169
+ dtype=dtype,
170
+ ).to(device)
171
+ # expand op needs to complete y+=a@lora_b, so output is
172
+ # initinized randomly
173
+ ref_out_tensor = torch.rand(
174
+ (total_tokens, hidden_size),
175
+ dtype=dtype,
176
+ ).to(device)
177
+ # Ensure the same input.
178
+ our_out_tensor = ref_out_tensor.clone()
179
+ lora_indices_tensor = torch.randint(
180
+ 0, lora_nums - 1 if lora_nums > 1 else 1, (batches,)
181
+ ).to(device)
182
+ indices = torch.zeros((total_tokens), dtype=torch.long).to(device)
183
+ current_offset = 0
184
+ for b_id in range(batches):
185
+ lora_index = lora_indices_tensor[b_id]
186
+ indices[current_offset : current_offset + seq_len_tensor[b_id]].copy_(
187
+ lora_index
188
+ )
189
+ current_offset += seq_len_tensor[b_id].item()
190
+
191
+ return PunicaTensors(
192
+ inputs_tensor,
193
+ lora_weights,
194
+ our_out_tensor,
195
+ ref_out_tensor,
196
+ b_seq_start_loc,
197
+ lora_indices_tensor,
198
+ seq_len_tensor,
199
+ indices,
200
+ )
201
+
202
+
203
+ def generate_data_for_expand_nslices(
204
+ batches,
205
+ hidden_size,
206
+ lora_nums,
207
+ max_rank,
208
+ seq_length,
209
+ dtype,
210
+ nslices,
211
+ device,
212
+ ) -> PunicaTensors:
213
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1, (batches,)).to(device)
214
+ b_seq_start_loc = torch.cumsum(
215
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
216
+ dim=0,
217
+ ).to(device)
218
+ total_tokens = seq_len_tensor.sum()
219
+ inputs_tensor = torch.rand(
220
+ (total_tokens, max_rank),
221
+ dtype=dtype,
222
+ ).to(device)
223
+ lora_weights_lst = []
224
+ for _ in range(nslices):
225
+ lora_weights_lst.append(
226
+ torch.rand(
227
+ (lora_nums, hidden_size, max_rank), # col-major
228
+ dtype=dtype,
229
+ ).to(device)
230
+ )
231
+ # expand op needs to complete y+=a@lora_b, so output is
232
+ # initinized randomly
233
+ ref_out_tensor = torch.rand((total_tokens, hidden_size * nslices), dtype=dtype).to(
234
+ device
235
+ )
236
+ # Ensure the same input.
237
+ our_out_tensor = ref_out_tensor.clone()
238
+ lora_indices_tensor = torch.randint(
239
+ 0, lora_nums - 1 if lora_nums > 1 else 1, (batches,)
240
+ )
241
+ indices = torch.zeros((total_tokens), dtype=torch.long).to(device)
242
+ current_offset = 0
243
+ for b_id in range(batches):
244
+ lora_index = lora_indices_tensor[b_id]
245
+ indices[current_offset : current_offset + seq_len_tensor[b_id]] = (
246
+ lora_index.item()
247
+ )
248
+ current_offset += seq_len_tensor[b_id].item()
249
+
250
+ lora_indices_tensor = lora_indices_tensor.to(device)
251
+ return PunicaTensors(
252
+ inputs_tensor,
253
+ lora_weights_lst,
254
+ our_out_tensor,
255
+ ref_out_tensor,
256
+ b_seq_start_loc,
257
+ lora_indices_tensor,
258
+ seq_len_tensor,
259
+ indices,
260
+ )
261
+
262
+
263
+ def generate_data_for_nslices(
264
+ batches,
265
+ hidden_size,
266
+ lora_nums,
267
+ max_rank,
268
+ seq_length,
269
+ nslices,
270
+ dtype,
271
+ op_type,
272
+ device,
273
+ ) -> PunicaTensors:
274
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1, (batches,)).to(device)
275
+ b_seq_start_loc = torch.cumsum(
276
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
277
+ dim=0,
278
+ ).to(device)
279
+ total_tokens = seq_len_tensor.sum()
280
+
281
+ lora_weights_lst = []
282
+ if op_type == "shrink":
283
+ inputs_tensor = torch.rand((total_tokens, hidden_size), dtype=dtype).to(device)
284
+
285
+ for _ in range(nslices):
286
+ if op_type == "shrink":
287
+ lora_weights_lst.append(
288
+ torch.rand(
289
+ (lora_nums, max_rank, hidden_size), # col-major
290
+ dtype=dtype,
291
+ ).to(device)
292
+ )
293
+ # NOTE shrink kernel using torch.float32 as output type
294
+ # shrink op need atomic_add, so output is initinized by 0
295
+ our_out_tensor = torch.zeros(
296
+ (nslices, total_tokens, max_rank),
297
+ dtype=torch.float32,
298
+ ).to(device)
299
+ else:
300
+ inputs_tensor = torch.rand(
301
+ (nslices, total_tokens, max_rank),
302
+ dtype=dtype,
303
+ ).to(device)
304
+ for _ in range(nslices):
305
+ lora_weights_lst.append(
306
+ torch.rand(
307
+ (lora_nums, hidden_size, max_rank), # col-major
308
+ dtype=dtype,
309
+ ).to(device)
310
+ )
311
+ # expand op needs to complete y+=a@lora_b, so output is
312
+ # initinized randomly
313
+ our_out_tensor = torch.rand(
314
+ (total_tokens, hidden_size * nslices), dtype=dtype
315
+ ).to(device)
316
+
317
+ # Ensure the same input.
318
+ ref_out_tensor = our_out_tensor.clone()
319
+ lora_indices_tensor = torch.randint(
320
+ 0, lora_nums - 1 if lora_nums > 1 else 1, (batches,)
321
+ )
322
+ indices = torch.zeros((total_tokens), dtype=torch.long).to(device)
323
+ current_offset = 0
324
+ for b_id in range(batches):
325
+ lora_index = lora_indices_tensor[b_id]
326
+ indices[current_offset : current_offset + seq_len_tensor[b_id]] = (
327
+ lora_index.item()
328
+ )
329
+ current_offset += seq_len_tensor[b_id].item()
330
+
331
+ lora_indices_tensor = lora_indices_tensor.to(device)
332
+ return PunicaTensors(
333
+ inputs_tensor,
334
+ lora_weights_lst,
335
+ our_out_tensor,
336
+ ref_out_tensor,
337
+ b_seq_start_loc,
338
+ lora_indices_tensor,
339
+ seq_len_tensor,
340
+ indices,
341
+ )
342
+
343
+
344
+ def create_peft_lora(
345
+ model: torch.nn.Module,
346
+ save_dir: str,
347
+ target_modules: list[str],
348
+ rank: int = 8,
349
+ alpha: int = 16,
350
+ dropout: float = 0.1,
351
+ lora_dtype: torch.dtype = torch.float16,
352
+ ) -> dict[str, torch.Tensor]:
353
+ lora_weights = {}
354
+ adapter_config = {
355
+ "peft_type": "LORA",
356
+ "auto_mapping": None,
357
+ "base_model_name_or_path": "dummy_model",
358
+ "revision": None,
359
+ "task_type": "CAUSAL_LM",
360
+ "inference_mode": False,
361
+ "r": rank,
362
+ "lora_alpha": alpha,
363
+ "lora_dropout": dropout,
364
+ "fan_in_fan_out": False,
365
+ "bias": "none",
366
+ "modules_to_save": None,
367
+ "init_lora_weights": True,
368
+ "layers_to_transform": None,
369
+ "layers_pattern": None,
370
+ "target_modules": target_modules,
371
+ "exclude_modules": None,
372
+ "use_rslora": False,
373
+ "use_dora": False,
374
+ "loftq_config": None,
375
+ }
376
+
377
+ for module_name in target_modules:
378
+ module = model
379
+ for attr in module_name.split("."):
380
+ module = getattr(module, attr)
381
+
382
+ if hasattr(module, "input_size") and hasattr(module, "output_size"):
383
+ in_features = module.input_size
384
+ out_features = module.output_size
385
+
386
+ elif hasattr(module, "embedding_dim") and hasattr(module, "num_embeddings"):
387
+ # ParallelLMHead
388
+ in_features = module.embedding_dim
389
+ out_features = module.num_embeddings
390
+ else:
391
+ raise ValueError(f"Unable to determine dimensions for module {module_name}")
392
+
393
+ lora_A = torch.randn(rank, in_features, dtype=lora_dtype)
394
+
395
+ torch.nn.init.kaiming_uniform_(lora_A, a=5**0.5)
396
+
397
+ lora_B = torch.zeros(out_features, rank, dtype=lora_dtype)
398
+
399
+ # PEFT style
400
+ lora_weights[f"base_model.model.{module_name}.lora_A.weight"] = lora_A
401
+ lora_weights[f"base_model.model.{module_name}.lora_B.weight"] = lora_B
402
+
403
+ config_path = os.path.join(save_dir, "adapter_config.json")
404
+ with open(config_path, "w", encoding="utf-8") as f:
405
+ json.dump(adapter_config, f, indent=2, ensure_ascii=False)
406
+
407
+ weights_path = os.path.join(save_dir, "adapter_model.safetensors")
408
+ save_file(lora_weights, weights_path)
409
+
410
+ return lora_weights
asian_ai/performance/vLLM/tests/model_executor/__init__.py ADDED
File without changes
asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_activations.py ADDED
@@ -0,0 +1,234 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ """Unit tests for vllm.model_executor.layers.pooler.activations."""
4
+
5
+ from types import SimpleNamespace
6
+
7
+ import pytest
8
+ import torch
9
+ import torch.nn as nn
10
+
11
+ from vllm.model_executor.layers.pooler.activations import (
12
+ LambdaPoolerActivation,
13
+ PoolerClassify,
14
+ PoolerIdentity,
15
+ PoolerMultiLabelClassify,
16
+ PoolerNormalize,
17
+ get_act_fn,
18
+ resolve_classifier_act_fn,
19
+ )
20
+
21
+
22
+ # ---------------------------------------------------------------------------
23
+ # PoolerIdentity
24
+ # ---------------------------------------------------------------------------
25
+ class TestPoolerIdentity:
26
+ def test_returns_input_unchanged(self):
27
+ pooler = PoolerIdentity()
28
+ x = torch.randn(4, 128)
29
+ out = pooler(x)
30
+ assert torch.equal(out, x)
31
+
32
+ def test_forward_list(self):
33
+ pooler = PoolerIdentity()
34
+ tensors = [torch.randn(128), torch.randn(256)]
35
+ out = pooler(tensors)
36
+ assert len(out) == 2
37
+ for orig, result in zip(tensors, out):
38
+ assert torch.equal(orig, result)
39
+
40
+
41
+ # ---------------------------------------------------------------------------
42
+ # PoolerNormalize
43
+ # ---------------------------------------------------------------------------
44
+ class TestPoolerNormalize:
45
+ def test_output_has_unit_norm(self):
46
+ pooler = PoolerNormalize()
47
+ x = torch.randn(4, 128)
48
+ out = pooler(x)
49
+ norms = torch.linalg.norm(out, dim=-1)
50
+ assert torch.allclose(norms, torch.ones(4), atol=1e-5)
51
+
52
+ def test_single_vector(self):
53
+ pooler = PoolerNormalize()
54
+ x = torch.randn(1, 64)
55
+ out = pooler(x)
56
+ norm = torch.linalg.norm(out, dim=-1)
57
+ assert torch.allclose(norm, torch.ones(1), atol=1e-5)
58
+
59
+ def test_forward_list(self):
60
+ pooler = PoolerNormalize()
61
+ tensors = [torch.randn(1, 64), torch.randn(1, 128)]
62
+ out = pooler(tensors)
63
+ for t in out:
64
+ norm = torch.linalg.norm(t, dim=-1)
65
+ assert torch.allclose(norm, torch.ones(1), atol=1e-5)
66
+
67
+
68
+ # ---------------------------------------------------------------------------
69
+ # PoolerMultiLabelClassify
70
+ # ---------------------------------------------------------------------------
71
+ class TestPoolerMultiLabelClassify:
72
+ def test_output_in_zero_one(self):
73
+ pooler = PoolerMultiLabelClassify()
74
+ x = torch.randn(4, 10)
75
+ out = pooler(x)
76
+ assert (out >= 0).all() and (out <= 1).all()
77
+
78
+ def test_large_positive_maps_near_one(self):
79
+ pooler = PoolerMultiLabelClassify()
80
+ x = torch.full((1, 3), 100.0)
81
+ out = pooler(x)
82
+ assert torch.allclose(out, torch.ones(1, 3), atol=1e-4)
83
+
84
+ def test_large_negative_maps_near_zero(self):
85
+ pooler = PoolerMultiLabelClassify()
86
+ x = torch.full((1, 3), -100.0)
87
+ out = pooler(x)
88
+ assert torch.allclose(out, torch.zeros(1, 3), atol=1e-4)
89
+
90
+
91
+ # ---------------------------------------------------------------------------
92
+ # PoolerClassify
93
+ # ---------------------------------------------------------------------------
94
+ class TestPoolerClassify:
95
+ def test_infers_from_shape_when_num_labels_none(self):
96
+ pooler = PoolerClassify(num_labels=None)
97
+ assert pooler.num_labels is None
98
+ x = torch.randn(2, 5)
99
+ out = pooler(x)
100
+ sums = out.sum(dim=-1)
101
+ assert torch.allclose(sums, torch.ones(2), atol=1e-5)
102
+
103
+ def test_sigmoid_when_num_labels_lt_2(self):
104
+ pooler = PoolerClassify(num_labels=1)
105
+ x = torch.zeros(1, 1)
106
+ out = pooler(x)
107
+ assert torch.allclose(out, torch.tensor([[0.5]]), atol=1e-5)
108
+
109
+ def test_num_labels_zero_uses_sigmoid(self):
110
+ pooler = PoolerClassify(num_labels=0)
111
+ assert pooler.num_labels == 0
112
+ x = torch.zeros(1, 3)
113
+ out = pooler(x)
114
+ assert torch.allclose(out, torch.full((1, 3), 0.5), atol=1e-5)
115
+
116
+ def test_num_labels_ge_2_uses_softmax(self):
117
+ pooler = PoolerClassify(num_labels=4)
118
+ assert pooler.num_labels == 4
119
+ x = torch.randn(2, 4)
120
+ out = pooler(x)
121
+ sums = out.sum(dim=-1)
122
+ assert torch.allclose(sums, torch.ones(2), atol=1e-5)
123
+
124
+ def test_default_num_labels_is_none(self):
125
+ pooler = PoolerClassify()
126
+ assert pooler.num_labels is None
127
+
128
+
129
+ # ---------------------------------------------------------------------------
130
+ # LambdaPoolerActivation
131
+ # ---------------------------------------------------------------------------
132
+ class TestLambdaPoolerActivation:
133
+ def test_applies_custom_fn(self):
134
+ pooler = LambdaPoolerActivation(nn.ReLU())
135
+ x = torch.tensor([[-1.0, 2.0, -3.0]])
136
+ out = pooler(x)
137
+ expected = torch.tensor([[0.0, 2.0, 0.0]])
138
+ assert torch.equal(out, expected)
139
+
140
+ def test_forward_list(self):
141
+ pooler = LambdaPoolerActivation(nn.ReLU())
142
+ tensors = [torch.tensor([-1.0, 2.0]), torch.tensor([3.0, -4.0])]
143
+ out = pooler(tensors)
144
+ assert torch.equal(out[0], torch.tensor([0.0, 2.0]))
145
+ assert torch.equal(out[1], torch.tensor([3.0, 0.0]))
146
+
147
+
148
+ # ---------------------------------------------------------------------------
149
+ # get_act_fn factory
150
+ # ---------------------------------------------------------------------------
151
+ class TestGetActFn:
152
+ @staticmethod
153
+ def _make_config(**kwargs):
154
+ return SimpleNamespace(**kwargs)
155
+
156
+ def test_regression(self):
157
+ cfg = self._make_config(problem_type="regression")
158
+ result = get_act_fn(cfg)
159
+ assert isinstance(result, PoolerIdentity)
160
+
161
+ def test_single_label_classification(self):
162
+ cfg = self._make_config(
163
+ problem_type="single_label_classification", num_labels=3
164
+ )
165
+ result = get_act_fn(cfg)
166
+ assert isinstance(result, PoolerClassify)
167
+ assert result.num_labels == 3
168
+
169
+ def test_multi_label_classification(self):
170
+ cfg = self._make_config(problem_type="multi_label_classification")
171
+ result = get_act_fn(cfg)
172
+ assert isinstance(result, PoolerMultiLabelClassify)
173
+
174
+ def test_sentence_transformers_activation(self):
175
+ cfg = self._make_config(
176
+ problem_type="",
177
+ sentence_transformers={
178
+ "activation_fn": "torch.nn.modules.activation.Sigmoid"
179
+ },
180
+ )
181
+ result = get_act_fn(cfg)
182
+ assert isinstance(result, PoolerClassify)
183
+
184
+ def test_sbert_activation(self):
185
+ cfg = self._make_config(
186
+ problem_type="",
187
+ sbert_ce_default_activation_function=(
188
+ "torch.nn.modules.activation.Sigmoid"
189
+ ),
190
+ )
191
+ result = get_act_fn(cfg)
192
+ assert isinstance(result, PoolerClassify)
193
+
194
+ def test_default_fallback(self):
195
+ cfg = self._make_config(problem_type="")
196
+ result = get_act_fn(cfg)
197
+ assert isinstance(result, PoolerClassify)
198
+
199
+ def test_sentence_transformers_takes_priority(self):
200
+ cfg = self._make_config(
201
+ problem_type="",
202
+ sentence_transformers={"activation_fn": "torch.nn.modules.linear.Identity"},
203
+ sbert_ce_default_activation_function=(
204
+ "torch.nn.modules.activation.Sigmoid"
205
+ ),
206
+ )
207
+ result = get_act_fn(cfg)
208
+ assert isinstance(result, PoolerIdentity)
209
+
210
+ def test_rejects_non_torch_activation(self):
211
+ cfg = self._make_config(
212
+ problem_type="",
213
+ sentence_transformers={"activation_fn": "os.system"},
214
+ )
215
+ with pytest.raises(ValueError, match="restricted"):
216
+ get_act_fn(cfg)
217
+
218
+
219
+ # ---------------------------------------------------------------------------
220
+ # resolve_classifier_act_fn
221
+ # ---------------------------------------------------------------------------
222
+ class TestResolveClassifierActFn:
223
+ def test_delegates_to_get_act_fn_when_none(self):
224
+ model_config = SimpleNamespace(
225
+ hf_config=SimpleNamespace(num_labels=3, problem_type="")
226
+ )
227
+ result = resolve_classifier_act_fn(model_config, act_fn=None)
228
+ assert isinstance(result, PoolerClassify)
229
+ assert result.num_labels == 3
230
+
231
+ def test_passes_through_provided_act_fn(self):
232
+ custom = PoolerIdentity()
233
+ result = resolve_classifier_act_fn(None, act_fn=custom)
234
+ assert result is custom
asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_heads.py ADDED
@@ -0,0 +1,481 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ """Unit tests for sequence and token pooler head classes."""
4
+
5
+ import torch
6
+ import torch.nn as nn
7
+
8
+ from vllm.model_executor.layers.pooler.activations import PoolerNormalize
9
+ from vllm.model_executor.layers.pooler.seqwise.heads import (
10
+ ClassifierPoolerHead,
11
+ EmbeddingPoolerHead,
12
+ )
13
+ from vllm.model_executor.layers.pooler.tokwise.heads import (
14
+ TokenClassifierPoolerHead,
15
+ TokenEmbeddingPoolerHead,
16
+ )
17
+ from vllm.pooling_params import PoolingParams
18
+ from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
19
+
20
+ _HIDDEN = 16
21
+ _BATCH = 3
22
+
23
+
24
+ def _make_params(
25
+ n: int,
26
+ *,
27
+ task: str = "embed",
28
+ dimensions: int | None = None,
29
+ use_activation: bool | None = None,
30
+ ) -> list[PoolingParams]:
31
+ return [
32
+ PoolingParams(task=task, dimensions=dimensions, use_activation=use_activation)
33
+ for _ in range(n)
34
+ ]
35
+
36
+
37
+ def _make_metadata(pooling_params: list[PoolingParams]) -> PoolingMetadata:
38
+ n = len(pooling_params)
39
+ return PoolingMetadata(
40
+ prompt_lens=torch.ones(n, dtype=torch.long),
41
+ prompt_token_ids=None,
42
+ prompt_token_ids_cpu=None,
43
+ pooling_params=pooling_params,
44
+ pooling_states=[PoolingStates() for _ in range(n)],
45
+ )
46
+
47
+
48
+ def _linear(in_f: int, out_f: int) -> nn.Linear:
49
+ torch.manual_seed(42)
50
+ return nn.Linear(in_f, out_f, bias=False)
51
+
52
+
53
+ # ---------------------------------------------------------------------------
54
+ # EmbeddingPoolerHead
55
+ # ---------------------------------------------------------------------------
56
+ class TestEmbeddingPoolerHead:
57
+ def test_supported_tasks(self):
58
+ head = EmbeddingPoolerHead()
59
+ assert head.get_supported_tasks() == {"embed"}
60
+
61
+ def test_passthrough(self):
62
+ head = EmbeddingPoolerHead()
63
+ x = torch.randn(_BATCH, _HIDDEN)
64
+ meta = _make_metadata(_make_params(_BATCH))
65
+ out = head(x, meta)
66
+ assert torch.equal(out, x)
67
+
68
+ def test_head_dtype(self):
69
+ head = EmbeddingPoolerHead(head_dtype=torch.float16)
70
+ x = torch.randn(_BATCH, _HIDDEN)
71
+ meta = _make_metadata(_make_params(_BATCH))
72
+ out = head(x, meta)
73
+ assert out.dtype == torch.float16
74
+
75
+ def test_projector(self):
76
+ proj = _linear(_HIDDEN, 8)
77
+ head = EmbeddingPoolerHead(projector=proj)
78
+ x = torch.randn(_BATCH, _HIDDEN)
79
+ meta = _make_metadata(_make_params(_BATCH))
80
+ out = head(x, meta)
81
+ assert out.shape == (_BATCH, 8)
82
+ assert torch.allclose(out, proj(x))
83
+
84
+ def test_matryoshka_uniform(self):
85
+ head = EmbeddingPoolerHead()
86
+ x = torch.randn(_BATCH, _HIDDEN)
87
+ params = _make_params(_BATCH, dimensions=4)
88
+ meta = _make_metadata(params)
89
+ out = head(x, meta)
90
+ assert out.shape == (_BATCH, 4)
91
+ assert torch.equal(out, x[..., :4])
92
+
93
+ def test_matryoshka_mixed(self):
94
+ head = EmbeddingPoolerHead()
95
+ x = torch.randn(2, _HIDDEN)
96
+ params = [
97
+ PoolingParams(task="embed", dimensions=4),
98
+ PoolingParams(task="embed", dimensions=8),
99
+ ]
100
+ meta = _make_metadata(params)
101
+ out = head(x, meta)
102
+ assert isinstance(out, list)
103
+ assert len(out) == 2
104
+ assert out[0].shape[-1] == 4
105
+ assert out[1].shape[-1] == 8
106
+
107
+ def test_matryoshka_mixed_with_none(self):
108
+ head = EmbeddingPoolerHead()
109
+ x = torch.randn(2, _HIDDEN)
110
+ params = [
111
+ PoolingParams(task="embed", dimensions=4),
112
+ PoolingParams(task="embed", dimensions=None),
113
+ ]
114
+ meta = _make_metadata(params)
115
+ out = head(x, meta)
116
+ assert isinstance(out, list)
117
+ assert out[0].shape[-1] == 4
118
+ assert torch.equal(out[1], x[1])
119
+
120
+ def test_activation_uniform_true(self):
121
+ head = EmbeddingPoolerHead(activation=PoolerNormalize())
122
+ x = torch.randn(_BATCH, _HIDDEN)
123
+ params = _make_params(_BATCH, use_activation=True)
124
+ meta = _make_metadata(params)
125
+ out = head(x, meta)
126
+ norms = torch.linalg.norm(out, dim=-1)
127
+ assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
128
+
129
+ def test_activation_uniform_false(self):
130
+ head = EmbeddingPoolerHead(activation=PoolerNormalize())
131
+ x = torch.randn(_BATCH, _HIDDEN)
132
+ params = _make_params(_BATCH, use_activation=False)
133
+ meta = _make_metadata(params)
134
+ out = head(x, meta)
135
+ assert torch.equal(out, x)
136
+
137
+ def test_activation_mixed_flags(self):
138
+ head = EmbeddingPoolerHead(activation=PoolerNormalize())
139
+ x = torch.randn(2, _HIDDEN)
140
+ params = [
141
+ PoolingParams(task="embed", use_activation=True),
142
+ PoolingParams(task="embed", use_activation=False),
143
+ ]
144
+ meta = _make_metadata(params)
145
+ out = head(x, meta)
146
+ assert isinstance(out, list)
147
+ norm_0 = torch.linalg.norm(out[0], dim=-1)
148
+ assert torch.allclose(norm_0, torch.ones(1), atol=1e-5)
149
+ assert torch.equal(out[1], x[1])
150
+
151
+ def test_list_input_gets_stacked(self):
152
+ head = EmbeddingPoolerHead()
153
+ tensors = [torch.randn(_HIDDEN) for _ in range(_BATCH)]
154
+ meta = _make_metadata(_make_params(_BATCH))
155
+ out = head(tensors, meta)
156
+ assert out.shape == (_BATCH, _HIDDEN)
157
+ expected = torch.stack(tensors)
158
+ assert torch.equal(out, expected)
159
+
160
+ def test_projector_then_matryoshka(self):
161
+ proj = _linear(_HIDDEN, 8)
162
+ head = EmbeddingPoolerHead(projector=proj)
163
+ x = torch.randn(_BATCH, _HIDDEN)
164
+ params = _make_params(_BATCH, dimensions=4)
165
+ meta = _make_metadata(params)
166
+ out = head(x, meta)
167
+ assert out.shape == (_BATCH, 4)
168
+ assert torch.equal(out, proj(x)[..., :4])
169
+
170
+ def test_matryoshka_then_activation(self):
171
+ head = EmbeddingPoolerHead(activation=PoolerNormalize())
172
+ x = torch.randn(_BATCH, _HIDDEN)
173
+ params = _make_params(_BATCH, dimensions=4, use_activation=True)
174
+ meta = _make_metadata(params)
175
+ out = head(x, meta)
176
+ assert out.shape == (_BATCH, 4)
177
+ norms = torch.linalg.norm(out, dim=-1)
178
+ assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
179
+
180
+ def test_empty_batch(self):
181
+ head = EmbeddingPoolerHead()
182
+ x = torch.randn(0, _HIDDEN)
183
+ meta = _make_metadata([])
184
+ out = head(x, meta)
185
+ assert out.shape == (0, _HIDDEN)
186
+
187
+
188
+ # ---------------------------------------------------------------------------
189
+ # ClassifierPoolerHead
190
+ # ---------------------------------------------------------------------------
191
+ class TestClassifierPoolerHead:
192
+ def test_supported_tasks(self):
193
+ head = ClassifierPoolerHead()
194
+ assert head.get_supported_tasks() == {"classify"}
195
+
196
+ def test_passthrough(self):
197
+ head = ClassifierPoolerHead()
198
+ x = torch.randn(_BATCH, _HIDDEN)
199
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
200
+ out = head(x, meta)
201
+ assert torch.equal(out, x)
202
+
203
+ def test_head_dtype(self):
204
+ head = ClassifierPoolerHead(head_dtype=torch.float16)
205
+ x = torch.randn(_BATCH, _HIDDEN)
206
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
207
+ out = head(x, meta)
208
+ assert out.dtype == torch.float16
209
+
210
+ def test_classifier(self):
211
+ clf = _linear(_HIDDEN, 3)
212
+ head = ClassifierPoolerHead(classifier=clf)
213
+ x = torch.randn(_BATCH, _HIDDEN)
214
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
215
+ out = head(x, meta)
216
+ assert out.shape == (_BATCH, 3)
217
+ assert torch.allclose(out, clf(x))
218
+
219
+ def test_logit_mean(self):
220
+ head = ClassifierPoolerHead(logit_mean=2.0)
221
+ x = torch.randn(_BATCH, _HIDDEN)
222
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
223
+ out = head(x, meta)
224
+ assert torch.allclose(out, x - 2.0)
225
+
226
+ def test_logit_sigma(self):
227
+ head = ClassifierPoolerHead(logit_sigma=0.5)
228
+ x = torch.randn(_BATCH, _HIDDEN)
229
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
230
+ out = head(x, meta)
231
+ assert torch.allclose(out, x / 0.5)
232
+
233
+ def test_platt_scaling_combined(self):
234
+ head = ClassifierPoolerHead(logit_mean=1.0, logit_sigma=2.0)
235
+ x = torch.randn(_BATCH, _HIDDEN)
236
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
237
+ out = head(x, meta)
238
+ assert torch.allclose(out, (x - 1.0) / 2.0)
239
+
240
+ def test_activation_uniform_true(self):
241
+ head = ClassifierPoolerHead(activation=PoolerNormalize())
242
+ x = torch.randn(_BATCH, _HIDDEN)
243
+ params = _make_params(_BATCH, task="classify", use_activation=True)
244
+ meta = _make_metadata(params)
245
+ out = head(x, meta)
246
+ norms = torch.linalg.norm(out, dim=-1)
247
+ assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
248
+
249
+ def test_activation_uniform_false(self):
250
+ head = ClassifierPoolerHead(activation=PoolerNormalize())
251
+ x = torch.randn(_BATCH, _HIDDEN)
252
+ params = _make_params(_BATCH, task="classify", use_activation=False)
253
+ meta = _make_metadata(params)
254
+ out = head(x, meta)
255
+ assert torch.equal(out, x)
256
+
257
+ def test_activation_mixed_flags(self):
258
+ head = ClassifierPoolerHead(activation=PoolerNormalize())
259
+ x = torch.randn(2, _HIDDEN)
260
+ params = [
261
+ PoolingParams(task="classify", use_activation=True),
262
+ PoolingParams(task="classify", use_activation=False),
263
+ ]
264
+ meta = _make_metadata(params)
265
+ out = head(x, meta)
266
+ assert isinstance(out, list)
267
+ norm_0 = torch.linalg.norm(out[0], dim=-1)
268
+ assert torch.allclose(norm_0, torch.ones(1), atol=1e-5)
269
+ assert torch.equal(out[1], x[1])
270
+
271
+ def test_list_input_gets_stacked(self):
272
+ head = ClassifierPoolerHead()
273
+ tensors = [torch.randn(_HIDDEN) for _ in range(_BATCH)]
274
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
275
+ out = head(tensors, meta)
276
+ assert out.shape == (_BATCH, _HIDDEN)
277
+ expected = torch.stack(tensors)
278
+ assert torch.equal(out, expected)
279
+
280
+ def test_classifier_then_platt_scaling(self):
281
+ clf = _linear(_HIDDEN, 3)
282
+ head = ClassifierPoolerHead(classifier=clf, logit_mean=1.0, logit_sigma=2.0)
283
+ x = torch.randn(_BATCH, _HIDDEN)
284
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
285
+ out = head(x, meta)
286
+ expected = (clf(x) - 1.0) / 2.0
287
+ assert torch.allclose(out, expected)
288
+
289
+ def test_empty_batch(self):
290
+ head = ClassifierPoolerHead()
291
+ x = torch.randn(0, _HIDDEN)
292
+ meta = _make_metadata([])
293
+ out = head(x, meta)
294
+ assert out.shape == (0, _HIDDEN)
295
+
296
+
297
+ # ---------------------------------------------------------------------------
298
+ # TokenEmbeddingPoolerHead
299
+ # ---------------------------------------------------------------------------
300
+ class TestTokenEmbeddingPoolerHead:
301
+ def test_supported_tasks(self):
302
+ head = TokenEmbeddingPoolerHead()
303
+ assert head.get_supported_tasks() == {"token_embed"}
304
+
305
+ def test_passthrough(self):
306
+ head = TokenEmbeddingPoolerHead()
307
+ x = torch.randn(5, _HIDDEN)
308
+ param = PoolingParams(task="token_embed")
309
+ out = head.forward_chunk(x, param)
310
+ assert torch.equal(out, x)
311
+
312
+ def test_none_chunked_prefill(self):
313
+ head = TokenEmbeddingPoolerHead()
314
+ param = PoolingParams(task="token_embed")
315
+ out = head.forward_chunk(None, param)
316
+ assert out is None
317
+
318
+ def test_head_dtype(self):
319
+ head = TokenEmbeddingPoolerHead(head_dtype=torch.float16)
320
+ x = torch.randn(5, _HIDDEN)
321
+ param = PoolingParams(task="token_embed")
322
+ out = head.forward_chunk(x, param)
323
+ assert out.dtype == torch.float16
324
+
325
+ def test_projector(self):
326
+ proj = _linear(_HIDDEN, 8)
327
+ head = TokenEmbeddingPoolerHead(projector=proj)
328
+ x = torch.randn(5, _HIDDEN)
329
+ param = PoolingParams(task="token_embed")
330
+ out = head.forward_chunk(x, param)
331
+ assert out.shape == (5, 8)
332
+ assert torch.allclose(out, proj(x))
333
+
334
+ def test_matryoshka_truncation(self):
335
+ head = TokenEmbeddingPoolerHead()
336
+ x = torch.randn(5, _HIDDEN)
337
+ param = PoolingParams(task="token_embed", dimensions=4)
338
+ out = head.forward_chunk(x, param)
339
+ assert out.shape == (5, 4)
340
+ assert torch.equal(out, x[..., :4])
341
+
342
+ def test_activation_true(self):
343
+ head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
344
+ x = torch.randn(5, _HIDDEN)
345
+ param = PoolingParams(task="token_embed", use_activation=True)
346
+ out = head.forward_chunk(x, param)
347
+ norms = torch.linalg.norm(out, dim=-1)
348
+ assert torch.allclose(norms, torch.ones(5), atol=1e-5)
349
+
350
+ def test_activation_false(self):
351
+ head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
352
+ x = torch.randn(5, _HIDDEN)
353
+ param = PoolingParams(task="token_embed", use_activation=False)
354
+ out = head.forward_chunk(x, param)
355
+ assert torch.equal(out, x)
356
+
357
+ def test_projector_then_matryoshka(self):
358
+ proj = _linear(_HIDDEN, 8)
359
+ head = TokenEmbeddingPoolerHead(projector=proj)
360
+ x = torch.randn(5, _HIDDEN)
361
+ param = PoolingParams(task="token_embed", dimensions=4)
362
+ out = head.forward_chunk(x, param)
363
+ assert out.shape == (5, 4)
364
+ assert torch.equal(out, proj(x)[..., :4])
365
+
366
+ def test_matryoshka_then_activation(self):
367
+ head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
368
+ x = torch.randn(5, _HIDDEN)
369
+ param = PoolingParams(task="token_embed", dimensions=4, use_activation=True)
370
+ out = head.forward_chunk(x, param)
371
+ assert out.shape == (5, 4)
372
+ norms = torch.linalg.norm(out, dim=-1)
373
+ assert torch.allclose(norms, torch.ones(5), atol=1e-5)
374
+
375
+ def test_forward_mixed_batch_chunked_prefill(self):
376
+ head = TokenEmbeddingPoolerHead()
377
+ pooled_data = [torch.randn(5, _HIDDEN), None, torch.randn(3, _HIDDEN)]
378
+ params = _make_params(3, task="token_embed")
379
+ meta = _make_metadata(params)
380
+ out = head(pooled_data, meta)
381
+ assert len(out) == 3
382
+ assert torch.equal(out[0], pooled_data[0])
383
+ assert out[1] is None
384
+ assert torch.equal(out[2], pooled_data[2])
385
+
386
+ def test_forward_empty_batch(self):
387
+ head = TokenEmbeddingPoolerHead()
388
+ meta = _make_metadata([])
389
+ out = head([], meta)
390
+ assert out == []
391
+
392
+
393
+ # ---------------------------------------------------------------------------
394
+ # TokenClassifierPoolerHead
395
+ # ---------------------------------------------------------------------------
396
+ class TestTokenClassifierPoolerHead:
397
+ def test_supported_tasks(self):
398
+ head = TokenClassifierPoolerHead()
399
+ assert head.get_supported_tasks() == {"token_classify"}
400
+
401
+ def test_passthrough(self):
402
+ head = TokenClassifierPoolerHead()
403
+ x = torch.randn(5, _HIDDEN)
404
+ param = PoolingParams(task="token_classify")
405
+ out = head.forward_chunk(x, param)
406
+ assert torch.equal(out, x)
407
+
408
+ def test_none_chunked_prefill(self):
409
+ head = TokenClassifierPoolerHead()
410
+ param = PoolingParams(task="token_classify")
411
+ out = head.forward_chunk(None, param)
412
+ assert out is None
413
+
414
+ def test_head_dtype(self):
415
+ head = TokenClassifierPoolerHead(head_dtype=torch.float16)
416
+ x = torch.randn(5, _HIDDEN)
417
+ param = PoolingParams(task="token_classify")
418
+ out = head.forward_chunk(x, param)
419
+ assert out.dtype == torch.float16
420
+
421
+ def test_classifier(self):
422
+ clf = _linear(_HIDDEN, 3)
423
+ head = TokenClassifierPoolerHead(classifier=clf)
424
+ x = torch.randn(5, _HIDDEN)
425
+ param = PoolingParams(task="token_classify")
426
+ out = head.forward_chunk(x, param)
427
+ assert out.shape == (5, 3)
428
+ assert torch.allclose(out, clf(x))
429
+
430
+ def test_logit_mean(self):
431
+ head = TokenClassifierPoolerHead(logit_mean=2.0)
432
+ x = torch.randn(5, _HIDDEN)
433
+ param = PoolingParams(task="token_classify")
434
+ out = head.forward_chunk(x, param)
435
+ assert torch.allclose(out, x - 2.0)
436
+
437
+ def test_logit_sigma(self):
438
+ head = TokenClassifierPoolerHead(logit_sigma=0.5)
439
+ x = torch.randn(5, _HIDDEN)
440
+ param = PoolingParams(task="token_classify")
441
+ out = head.forward_chunk(x, param)
442
+ assert torch.allclose(out, x / 0.5)
443
+
444
+ def test_platt_scaling_combined(self):
445
+ head = TokenClassifierPoolerHead(logit_mean=1.0, logit_sigma=2.0)
446
+ x = torch.randn(5, _HIDDEN)
447
+ param = PoolingParams(task="token_classify")
448
+ out = head.forward_chunk(x, param)
449
+ assert torch.allclose(out, (x - 1.0) / 2.0)
450
+
451
+ def test_activation_true(self):
452
+ head = TokenClassifierPoolerHead(activation=PoolerNormalize())
453
+ x = torch.randn(5, _HIDDEN)
454
+ param = PoolingParams(task="token_classify", use_activation=True)
455
+ out = head.forward_chunk(x, param)
456
+ norms = torch.linalg.norm(out, dim=-1)
457
+ assert torch.allclose(norms, torch.ones(5), atol=1e-5)
458
+
459
+ def test_activation_false(self):
460
+ head = TokenClassifierPoolerHead(activation=PoolerNormalize())
461
+ x = torch.randn(5, _HIDDEN)
462
+ param = PoolingParams(task="token_classify", use_activation=False)
463
+ out = head.forward_chunk(x, param)
464
+ assert torch.equal(out, x)
465
+
466
+ def test_forward_mixed_batch_chunked_prefill(self):
467
+ head = TokenClassifierPoolerHead()
468
+ pooled_data = [torch.randn(5, _HIDDEN), None, torch.randn(3, _HIDDEN)]
469
+ params = _make_params(3, task="token_classify")
470
+ meta = _make_metadata(params)
471
+ out = head(pooled_data, meta)
472
+ assert len(out) == 3
473
+ assert torch.equal(out[0], pooled_data[0])
474
+ assert out[1] is None
475
+ assert torch.equal(out[2], pooled_data[2])
476
+
477
+ def test_forward_empty_batch(self):
478
+ head = TokenClassifierPoolerHead()
479
+ meta = _make_metadata([])
480
+ out = head([], meta)
481
+ assert out == []
asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_methods.py ADDED
@@ -0,0 +1,499 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ """Unit tests for sequence and token pooling methods and their factories."""
4
+
5
+ from dataclasses import dataclass
6
+ from unittest.mock import patch
7
+
8
+ import pytest
9
+ import torch
10
+
11
+ from vllm.model_executor.layers.pooler.seqwise.methods import (
12
+ CLSPool,
13
+ LastPool,
14
+ MeanPool,
15
+ get_seq_pooling_method,
16
+ )
17
+ from vllm.model_executor.layers.pooler.tokwise.methods import (
18
+ AllPool,
19
+ StepPool,
20
+ get_tok_pooling_method,
21
+ )
22
+ from vllm.pooling_params import PoolingParams
23
+ from vllm.v1.pool.metadata import PoolingCursor, PoolingMetadata, PoolingStates
24
+
25
+ _CPU = torch.device("cpu")
26
+
27
+
28
+ def _make_pooling_cursor(
29
+ prompt_lens: list[int],
30
+ *,
31
+ num_scheduled_tokens: list[int] | None = None,
32
+ seq_lens: list[int] | None = None,
33
+ device: torch.device = _CPU,
34
+ ) -> PoolingCursor:
35
+ """Build a PoolingCursor from a list of per-sequence prompt lengths."""
36
+ prompt_lens_cpu = torch.tensor(prompt_lens, dtype=torch.long)
37
+ if num_scheduled_tokens is None:
38
+ num_scheduled_tokens_cpu = prompt_lens_cpu.clone()
39
+ else:
40
+ num_scheduled_tokens_cpu = torch.tensor(num_scheduled_tokens, dtype=torch.long)
41
+ if seq_lens is None:
42
+ seq_lens_cpu = prompt_lens_cpu.clone()
43
+ else:
44
+ seq_lens_cpu = torch.tensor(seq_lens, dtype=torch.long)
45
+
46
+ cumsum = torch.zeros(len(prompt_lens) + 1, dtype=torch.long, device=device)
47
+ torch.cumsum(num_scheduled_tokens_cpu, dim=0, out=cumsum[1:])
48
+
49
+ return PoolingCursor(
50
+ first_token_indices_gpu=cumsum[: len(prompt_lens)].to(device),
51
+ last_token_indices_gpu=(cumsum[1:] - 1).to(device),
52
+ prompt_lens_cpu=prompt_lens_cpu,
53
+ seq_lens_cpu=seq_lens_cpu,
54
+ num_scheduled_tokens_cpu=num_scheduled_tokens_cpu,
55
+ )
56
+
57
+
58
+ def _make_metadata(
59
+ prompt_lens: list[int],
60
+ *,
61
+ tasks: list[str] | None = None,
62
+ token_ids: list[list[int]] | None = None,
63
+ pooling_params: list[PoolingParams] | None = None,
64
+ num_scheduled_tokens: list[int] | None = None,
65
+ seq_lens: list[int] | None = None,
66
+ device: torch.device = _CPU,
67
+ ) -> PoolingMetadata:
68
+ """Build a minimal PoolingMetadata for testing pooling methods."""
69
+ n_seqs = len(prompt_lens)
70
+ if tasks is None:
71
+ tasks = ["embed"] * n_seqs
72
+ if pooling_params is None:
73
+ pooling_params = [PoolingParams(task=t) for t in tasks]
74
+
75
+ prompt_lens_tensor = torch.tensor(prompt_lens, dtype=torch.long)
76
+
77
+ prompt_token_ids_cpu = None
78
+ prompt_token_ids = None
79
+ if token_ids is not None:
80
+ max_len = max(len(t) for t in token_ids)
81
+ padded = [t + [0] * (max_len - len(t)) for t in token_ids]
82
+ prompt_token_ids_cpu = torch.tensor(padded, dtype=torch.long)
83
+ prompt_token_ids = prompt_token_ids_cpu.to(device)
84
+
85
+ cursor = _make_pooling_cursor(
86
+ prompt_lens,
87
+ num_scheduled_tokens=num_scheduled_tokens,
88
+ seq_lens=seq_lens,
89
+ device=device,
90
+ )
91
+
92
+ pooling_states = [PoolingStates() for _ in range(n_seqs)]
93
+
94
+ return PoolingMetadata(
95
+ prompt_lens=prompt_lens_tensor,
96
+ prompt_token_ids=prompt_token_ids,
97
+ prompt_token_ids_cpu=prompt_token_ids_cpu,
98
+ pooling_params=pooling_params,
99
+ pooling_states=pooling_states,
100
+ pooling_cursor=cursor,
101
+ )
102
+
103
+
104
+ # ---------------------------------------------------------------------------
105
+ # CLSPool
106
+ # ---------------------------------------------------------------------------
107
+ class TestCLSPool:
108
+ def test_extracts_first_token(self):
109
+ hidden = torch.tensor(
110
+ [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
111
+ )
112
+ metadata = _make_metadata([2, 3])
113
+ pooler = CLSPool()
114
+ out = pooler(hidden, metadata)
115
+ expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
116
+ assert torch.equal(out, expected)
117
+
118
+ def test_rejects_partial_prefill(self):
119
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
120
+ metadata = _make_metadata([3], num_scheduled_tokens=[2])
121
+ pooler = CLSPool()
122
+ with pytest.raises(RuntimeError, match="partial prefill"):
123
+ pooler(hidden, metadata)
124
+
125
+
126
+ # ---------------------------------------------------------------------------
127
+ # LastPool
128
+ # ---------------------------------------------------------------------------
129
+ class TestLastPool:
130
+ def test_extracts_last_token(self):
131
+ hidden = torch.tensor(
132
+ [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
133
+ )
134
+ metadata = _make_metadata([2, 3])
135
+ pooler = LastPool()
136
+ out = pooler(hidden, metadata)
137
+ expected = torch.tensor([[3.0, 4.0], [9.0, 10.0]])
138
+ assert torch.equal(out, expected)
139
+
140
+ def test_partial_prefill_extracts_last_scheduled(self):
141
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
142
+ metadata = _make_metadata([4], num_scheduled_tokens=[2])
143
+ pooler = LastPool()
144
+ out = pooler(hidden, metadata)
145
+ expected = torch.tensor([[3.0, 4.0]])
146
+ assert torch.equal(out, expected)
147
+
148
+
149
+ # ---------------------------------------------------------------------------
150
+ # MeanPool
151
+ # ---------------------------------------------------------------------------
152
+ class TestMeanPool:
153
+ def test_computes_mean(self):
154
+ hidden = torch.tensor(
155
+ [[1.0, 2.0], [3.0, 4.0], [10.0, 20.0]], dtype=torch.float32
156
+ )
157
+ metadata = _make_metadata([2, 1])
158
+ pooler = MeanPool()
159
+ out = pooler(hidden, metadata)
160
+ expected = torch.tensor([[2.0, 3.0], [10.0, 20.0]], dtype=torch.float32)
161
+ assert torch.allclose(out, expected, atol=1e-5)
162
+
163
+ def test_single_token_is_identity(self):
164
+ hidden = torch.tensor([[5.0, 10.0]], dtype=torch.float32)
165
+ metadata = _make_metadata([1])
166
+ pooler = MeanPool()
167
+ out = pooler(hidden, metadata)
168
+ assert torch.allclose(out, hidden, atol=1e-5)
169
+
170
+ def test_uniform_values_return_same(self):
171
+ hidden = torch.full((4, 3), 7.0, dtype=torch.float32)
172
+ metadata = _make_metadata([4])
173
+ pooler = MeanPool()
174
+ out = pooler(hidden, metadata)
175
+ expected = torch.full((1, 3), 7.0, dtype=torch.float32)
176
+ assert torch.allclose(out, expected, atol=1e-5)
177
+
178
+ def test_multiple_sequences(self):
179
+ hidden = torch.tensor(
180
+ [
181
+ [0.0, 0.0],
182
+ [2.0, 4.0],
183
+ [4.0, 8.0],
184
+ [10.0, 10.0],
185
+ ],
186
+ dtype=torch.float32,
187
+ )
188
+ metadata = _make_metadata([3, 1])
189
+ pooler = MeanPool()
190
+ out = pooler(hidden, metadata)
191
+ expected = torch.tensor([[2.0, 4.0], [10.0, 10.0]], dtype=torch.float32)
192
+ assert torch.allclose(out, expected, atol=1e-5)
193
+
194
+ def test_empty_batch(self):
195
+ hidden = torch.empty((0, 8), dtype=torch.float32)
196
+ metadata = _make_metadata([])
197
+ pooler = MeanPool()
198
+ out = pooler(hidden, metadata)
199
+ assert out.shape == (0, 8)
200
+
201
+ def test_rejects_partial_prefill(self):
202
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float32)
203
+ metadata = _make_metadata([3], num_scheduled_tokens=[2])
204
+ pooler = MeanPool()
205
+ with pytest.raises(RuntimeError, match="partial prefill"):
206
+ pooler(hidden, metadata)
207
+
208
+ def test_chunked_accumulation(self):
209
+ hidden = torch.arange(20, dtype=torch.float32).reshape(5, 4)
210
+ metadata = _make_metadata([3, 2])
211
+ pooler = MeanPool()
212
+ with patch(
213
+ "vllm.model_executor.layers.pooler.seqwise.methods"
214
+ "._MEAN_POOL_ACCUMULATION_CHUNK_BYTES",
215
+ 16,
216
+ ):
217
+ out = pooler(hidden, metadata)
218
+ expected_seq0 = hidden[:3].float().mean(dim=0, keepdim=True)
219
+ expected_seq1 = hidden[3:].float().mean(dim=0, keepdim=True)
220
+ expected = torch.cat([expected_seq0, expected_seq1], dim=0)
221
+ assert torch.allclose(out, expected, atol=1e-5)
222
+
223
+ def test_upcasts_to_float32(self):
224
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float16)
225
+ metadata = _make_metadata([2])
226
+ pooler = MeanPool()
227
+ out = pooler(hidden, metadata)
228
+ assert out.dtype == torch.float32
229
+ expected = torch.tensor([[2.0, 3.0]], dtype=torch.float32)
230
+ assert torch.allclose(out, expected, atol=1e-2)
231
+
232
+
233
+ # ---------------------------------------------------------------------------
234
+ # get_seq_pooling_method factory
235
+ # ---------------------------------------------------------------------------
236
+ class TestGetSeqPoolingMethod:
237
+ def test_cls(self):
238
+ assert isinstance(get_seq_pooling_method("CLS"), CLSPool)
239
+
240
+ def test_last(self):
241
+ assert isinstance(get_seq_pooling_method("LAST"), LastPool)
242
+
243
+ def test_mean(self):
244
+ assert isinstance(get_seq_pooling_method("MEAN"), MeanPool)
245
+
246
+ def test_unknown_raises(self):
247
+ with pytest.raises(NotImplementedError, match="UNKNOWN"):
248
+ get_seq_pooling_method("UNKNOWN")
249
+
250
+
251
+ # ---------------------------------------------------------------------------
252
+ # AllPool
253
+ # ---------------------------------------------------------------------------
254
+
255
+
256
+ @dataclass
257
+ class _FakeSchedulerConfig:
258
+ enable_chunked_prefill: bool = False
259
+
260
+
261
+ @dataclass
262
+ class _FakeVllmConfig:
263
+ scheduler_config: _FakeSchedulerConfig
264
+
265
+
266
+ class TestAllPool:
267
+ @staticmethod
268
+ def _make_all_pool(*, chunked: bool = False) -> AllPool:
269
+ fake_config = _FakeVllmConfig(
270
+ scheduler_config=_FakeSchedulerConfig(
271
+ enable_chunked_prefill=chunked,
272
+ ),
273
+ )
274
+ with patch(
275
+ "vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
276
+ return_value=fake_config,
277
+ ):
278
+ return AllPool()
279
+
280
+ def test_splits_by_sequence(self):
281
+ pooler = self._make_all_pool()
282
+ hidden = torch.tensor(
283
+ [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
284
+ )
285
+ metadata = _make_metadata([2, 3])
286
+ out = pooler(hidden, metadata)
287
+ assert len(out) == 2
288
+ assert torch.equal(out[0], hidden[:2])
289
+ assert torch.equal(out[1], hidden[2:])
290
+
291
+ def test_single_sequence(self):
292
+ pooler = self._make_all_pool()
293
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
294
+ metadata = _make_metadata([3])
295
+ out = pooler(hidden, metadata)
296
+ assert len(out) == 1
297
+ assert torch.equal(out[0], hidden)
298
+
299
+ def test_chunked_prefill_returns_none_for_unfinished(self):
300
+ pooler = self._make_all_pool(chunked=True)
301
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
302
+ metadata = _make_metadata(
303
+ [4],
304
+ num_scheduled_tokens=[2],
305
+ seq_lens=[2],
306
+ )
307
+ out = pooler(hidden, metadata)
308
+ assert len(out) == 1
309
+ assert out[0] is None
310
+
311
+ def test_chunked_prefill_returns_concat_when_finished(self):
312
+ pooler = self._make_all_pool(chunked=True)
313
+
314
+ chunk1 = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
315
+ metadata1 = _make_metadata(
316
+ [4],
317
+ num_scheduled_tokens=[2],
318
+ seq_lens=[2],
319
+ )
320
+ out1 = pooler(chunk1, metadata1)
321
+ assert out1[0] is None
322
+
323
+ chunk2 = torch.tensor([[5.0, 6.0], [7.0, 8.0]])
324
+ metadata2 = _make_metadata(
325
+ [4],
326
+ num_scheduled_tokens=[2],
327
+ seq_lens=[4],
328
+ )
329
+ metadata2.pooling_states = metadata1.pooling_states
330
+ out2 = pooler(chunk2, metadata2)
331
+ assert out2[0] is not None
332
+ expected = torch.cat([chunk1, chunk2], dim=0)
333
+ assert torch.equal(out2[0], expected)
334
+
335
+ def test_chunked_prefill_single_shot_matches_non_chunked(self):
336
+ pooler = self._make_all_pool(chunked=True)
337
+ hidden = torch.tensor(
338
+ [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
339
+ )
340
+ metadata = _make_metadata([2, 3])
341
+ out = pooler(hidden, metadata)
342
+ assert len(out) == 2
343
+ assert torch.equal(out[0], hidden[:2])
344
+ assert torch.equal(out[1], hidden[2:])
345
+
346
+ def test_chunked_prefill_mixed_finished_unfinished(self):
347
+ pooler = self._make_all_pool(chunked=True)
348
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
349
+ metadata = _make_metadata(
350
+ [2, 4],
351
+ num_scheduled_tokens=[2, 1],
352
+ seq_lens=[2, 1],
353
+ )
354
+ out = pooler(hidden, metadata)
355
+ assert len(out) == 2
356
+ assert torch.equal(out[0], hidden[:2])
357
+ assert out[1] is None
358
+
359
+
360
+ # ---------------------------------------------------------------------------
361
+ # StepPool
362
+ # ---------------------------------------------------------------------------
363
+ class TestStepPool:
364
+ @staticmethod
365
+ def _make_step_pool(*, chunked: bool = False) -> StepPool:
366
+ fake_config = _FakeVllmConfig(
367
+ scheduler_config=_FakeSchedulerConfig(
368
+ enable_chunked_prefill=chunked,
369
+ ),
370
+ )
371
+ with patch(
372
+ "vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
373
+ return_value=fake_config,
374
+ ):
375
+ return StepPool()
376
+
377
+ def test_filters_by_step_tag_id(self):
378
+ pooler = self._make_step_pool()
379
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
380
+ token_ids = [[10, 99, 10, 20]]
381
+ params = [PoolingParams(task="token_classify", step_tag_id=10)]
382
+ metadata = _make_metadata([4], token_ids=token_ids, pooling_params=params)
383
+ out = pooler(hidden, metadata)
384
+ assert len(out) == 1
385
+ expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
386
+ assert torch.equal(out[0], expected)
387
+
388
+ def test_filters_by_returned_token_ids(self):
389
+ pooler = self._make_step_pool()
390
+ hidden = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
391
+ token_ids = [[10, 20]]
392
+ params = [PoolingParams(task="token_classify", returned_token_ids=[0, 2])]
393
+ metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
394
+ out = pooler(hidden, metadata)
395
+ assert len(out) == 1
396
+ expected = torch.tensor([[1.0, 3.0], [4.0, 6.0]])
397
+ assert torch.equal(out[0], expected)
398
+
399
+ def test_no_filtering_without_params(self):
400
+ pooler = self._make_step_pool()
401
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
402
+ token_ids = [[10, 20]]
403
+ params = [PoolingParams(task="token_classify")]
404
+ metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
405
+ out = pooler(hidden, metadata)
406
+ assert len(out) == 1
407
+ assert torch.equal(out[0], hidden)
408
+
409
+ def test_combined_step_tag_and_returned_token_ids(self):
410
+ pooler = self._make_step_pool()
411
+ hidden = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]])
412
+ token_ids = [[99, 10, 99]]
413
+ params = [
414
+ PoolingParams(
415
+ task="token_classify",
416
+ step_tag_id=10,
417
+ returned_token_ids=[0, 2],
418
+ )
419
+ ]
420
+ metadata = _make_metadata([3], token_ids=token_ids, pooling_params=params)
421
+ out = pooler(hidden, metadata)
422
+ assert len(out) == 1
423
+ expected = torch.tensor([[4.0, 6.0]])
424
+ assert torch.equal(out[0], expected)
425
+
426
+ def test_step_tag_id_no_match_returns_empty(self):
427
+ pooler = self._make_step_pool()
428
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
429
+ token_ids = [[10, 20]]
430
+ params = [PoolingParams(task="token_classify", step_tag_id=999)]
431
+ metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
432
+ out = pooler(hidden, metadata)
433
+ assert len(out) == 1
434
+ assert out[0].shape == (0, 2)
435
+
436
+ def test_chunked_prefill_propagates_none_for_unfinished(self):
437
+ pooler = self._make_step_pool(chunked=True)
438
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
439
+ token_ids = [[10, 20, 30, 40]]
440
+ params = [PoolingParams(task="token_classify", step_tag_id=10)]
441
+ metadata = _make_metadata(
442
+ [4],
443
+ token_ids=token_ids,
444
+ pooling_params=params,
445
+ num_scheduled_tokens=[2],
446
+ seq_lens=[2],
447
+ )
448
+ out = pooler(hidden, metadata)
449
+ assert len(out) == 1
450
+ assert out[0] is None
451
+
452
+ def test_chunked_prefill_filters_when_finished(self):
453
+ pooler = self._make_step_pool(chunked=True)
454
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
455
+ token_ids = [[10, 99, 10, 20]]
456
+ params = [PoolingParams(task="token_classify", step_tag_id=10)]
457
+ metadata = _make_metadata([4], token_ids=token_ids, pooling_params=params)
458
+ out = pooler(hidden, metadata)
459
+ assert len(out) == 1
460
+ expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
461
+ assert torch.equal(out[0], expected)
462
+
463
+ def test_requires_token_ids_update(self):
464
+ pooler = self._make_step_pool()
465
+ update = pooler.get_pooling_updates("token_classify")
466
+ assert update.requires_token_ids is True
467
+
468
+
469
+ # ---------------------------------------------------------------------------
470
+ # get_tok_pooling_method factory
471
+ # ---------------------------------------------------------------------------
472
+ class TestGetTokPoolingMethod:
473
+ def test_all(self):
474
+ fake_config = _FakeVllmConfig(
475
+ scheduler_config=_FakeSchedulerConfig(
476
+ enable_chunked_prefill=False,
477
+ ),
478
+ )
479
+ with patch(
480
+ "vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
481
+ return_value=fake_config,
482
+ ):
483
+ assert isinstance(get_tok_pooling_method("ALL"), AllPool)
484
+
485
+ def test_step(self):
486
+ fake_config = _FakeVllmConfig(
487
+ scheduler_config=_FakeSchedulerConfig(
488
+ enable_chunked_prefill=False,
489
+ ),
490
+ )
491
+ with patch(
492
+ "vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
493
+ return_value=fake_config,
494
+ ):
495
+ assert isinstance(get_tok_pooling_method("STEP"), StepPool)
496
+
497
+ def test_unknown_raises(self):
498
+ with pytest.raises(NotImplementedError, match="UNKNOWN"):
499
+ get_tok_pooling_method("UNKNOWN")
asian_ai/performance/vLLM/tests/model_executor/layers/test_rocm_unquantized_gemm.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ from unittest.mock import MagicMock
5
+
6
+ import pytest
7
+ import torch
8
+
9
+ from vllm.platforms import current_platform
10
+
11
+ if current_platform.is_cuda():
12
+ pytest.skip(
13
+ "ROCm skinny GEMM tests are not supported on CUDA.",
14
+ allow_module_level=True,
15
+ )
16
+
17
+ from vllm.model_executor.layers import utils
18
+
19
+
20
+ def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch):
21
+ x = torch.randn(1, 64, dtype=torch.float16)
22
+ weight = torch.randn(128, 64, dtype=torch.float16)
23
+
24
+ monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
25
+ monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
26
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
27
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
28
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
29
+ monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
30
+
31
+ wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
32
+ monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
33
+ llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
34
+ monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
35
+
36
+ out = utils.rocm_unquantized_gemm_impl(x, weight, None)
37
+ ref = torch.nn.functional.linear(x, weight, None)
38
+
39
+ wvsplitk_mock.assert_called_once()
40
+ llmm1_mock.assert_not_called()
41
+ assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
42
+
43
+
44
+ def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch):
45
+ # wvSplitK skinny GEMM handles n in [1, 5] (see PR #40687); n > 5 must
46
+ # fall back to torch.nn.functional.linear.
47
+ x = torch.randn(6, 64, dtype=torch.float16)
48
+ weight = torch.randn(128, 64, dtype=torch.float16)
49
+
50
+ monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
51
+ monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
52
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
53
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
54
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
55
+ monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
56
+
57
+ wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
58
+ monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
59
+ llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
60
+ monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
61
+
62
+ out = utils.rocm_unquantized_gemm_impl(x, weight, None)
63
+ ref = torch.nn.functional.linear(x, weight, None)
64
+
65
+ wvsplitk_mock.assert_not_called()
66
+ llmm1_mock.assert_not_called()
67
+ assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
68
+
69
+
70
+ def test_rocm_unquantized_gemm_gfx950_wvsplitkrc_path(monkeypatch):
71
+ x = torch.randn(16, 1024, dtype=torch.float16)
72
+ weight = torch.randn(256, 1024, dtype=torch.float16)
73
+
74
+ monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
75
+ monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
76
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: False)
77
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
78
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
79
+ monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
80
+
81
+ wvsplitkrc_mock = MagicMock(side_effect=lambda x_view, w, _, __: x_view @ w.t())
82
+ monkeypatch.setattr(utils.ops, "wvSplitKrc", wvsplitkrc_mock)
83
+ wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
84
+ monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
85
+
86
+ out = utils.rocm_unquantized_gemm_impl(x, weight, None)
87
+ ref = torch.nn.functional.linear(x, weight, None)
88
+
89
+ wvsplitkrc_mock.assert_called_once()
90
+ wvsplitk_mock.assert_not_called()
91
+ assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
asian_ai/performance/vLLM/tests/model_executor/model_loader/__init__.py ADDED
File without changes
asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/__init__.py ADDED
File without changes
asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/test_fastsafetensors_loader.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import pytest
5
+
6
+ from vllm import SamplingParams
7
+ from vllm.platforms import current_platform
8
+
9
+ test_model = "openai-community/gpt2"
10
+
11
+ prompts = [
12
+ "Hello, my name is",
13
+ "The president of the United States is",
14
+ "The capital of France is",
15
+ "The future of AI is",
16
+ ]
17
+ # Create a sampling params object.
18
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
19
+
20
+
21
+ @pytest.mark.skipif(
22
+ not current_platform.is_cuda_alike(),
23
+ reason="fastsafetensors requires NVIDIA/AMD GPUs",
24
+ )
25
+ def test_model_loader_download_files(vllm_runner):
26
+ with vllm_runner(test_model, load_format="fastsafetensors") as llm:
27
+ deserialized_outputs = llm.generate(prompts, sampling_params)
28
+ assert deserialized_outputs
asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/test_weight_utils.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import glob
5
+ import tempfile
6
+
7
+ import huggingface_hub.constants
8
+ import pytest
9
+ import torch
10
+
11
+ from vllm.model_executor.model_loader.weight_utils import (
12
+ download_weights_from_hf,
13
+ fastsafetensors_weights_iterator,
14
+ safetensors_weights_iterator,
15
+ )
16
+ from vllm.platforms import current_platform
17
+
18
+
19
+ @pytest.mark.skipif(
20
+ not current_platform.is_cuda_alike(),
21
+ reason="fastsafetensors requires NVIDIA/AMD GPUs",
22
+ )
23
+ @pytest.mark.parametrize("queue_size", [0, 1])
24
+ def test_fastsafetensors_model_loader(monkeypatch, queue_size):
25
+ monkeypatch.setenv("VLLM_FASTSAFETENSORS_QUEUE_SIZE", str(queue_size))
26
+ with tempfile.TemporaryDirectory() as tmpdir:
27
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
28
+ download_weights_from_hf(
29
+ "openai-community/gpt2", allow_patterns=["*.safetensors"], cache_dir=tmpdir
30
+ )
31
+ safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
32
+ assert len(safetensors) > 0
33
+
34
+ fastsafetensors_tensors = {}
35
+ hf_safetensors_tensors = {}
36
+
37
+ for name, tensor in fastsafetensors_weights_iterator(safetensors, True):
38
+ fastsafetensors_tensors[name] = tensor
39
+
40
+ for name, tensor in safetensors_weights_iterator(safetensors, True):
41
+ hf_safetensors_tensors[name] = tensor
42
+
43
+ assert len(fastsafetensors_tensors) == len(hf_safetensors_tensors)
44
+
45
+ for name, fastsafetensors_tensor in fastsafetensors_tensors.items():
46
+ fastsafetensors_tensor = fastsafetensors_tensor.to("cpu")
47
+ assert fastsafetensors_tensor.dtype == hf_safetensors_tensors[name].dtype
48
+ assert fastsafetensors_tensor.shape == hf_safetensors_tensors[name].shape
49
+ assert torch.all(fastsafetensors_tensor.eq(hf_safetensors_tensors[name]))
asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/__init__.py ADDED
File without changes
asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/test_instanttensor_loader.py ADDED
@@ -0,0 +1,28 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import pytest
5
+
6
+ from vllm import SamplingParams
7
+ from vllm.platforms import current_platform
8
+
9
+ test_model = "openai-community/gpt2"
10
+
11
+ prompts = [
12
+ "Hello, my name is",
13
+ "The president of the United States is",
14
+ "The capital of France is",
15
+ "The future of AI is",
16
+ ]
17
+ # Create a sampling params object.
18
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
19
+
20
+
21
+ @pytest.mark.skipif(
22
+ not current_platform.is_cuda(),
23
+ reason="InstantTensor requires NVIDIA GPUs",
24
+ )
25
+ def test_model_loader_download_files(vllm_runner):
26
+ with vllm_runner(test_model, load_format="instanttensor") as llm:
27
+ deserialized_outputs = llm.generate(prompts, sampling_params)
28
+ assert deserialized_outputs
asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/test_weight_utils.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import glob
5
+ import tempfile
6
+
7
+ import huggingface_hub.constants
8
+ import pytest
9
+ import torch
10
+
11
+ from vllm.model_executor.model_loader.weight_utils import (
12
+ download_weights_from_hf,
13
+ instanttensor_weights_iterator,
14
+ safetensors_weights_iterator,
15
+ )
16
+ from vllm.platforms import current_platform
17
+
18
+
19
+ @pytest.mark.skipif(
20
+ not current_platform.is_cuda(),
21
+ reason="InstantTensor requires NVIDIA GPUs",
22
+ )
23
+ def test_instanttensor_model_loader():
24
+ with tempfile.TemporaryDirectory() as tmpdir:
25
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
26
+ download_weights_from_hf(
27
+ "openai-community/gpt2", allow_patterns=["*.safetensors"], cache_dir=tmpdir
28
+ )
29
+ safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
30
+ assert len(safetensors) > 0
31
+
32
+ instanttensor_tensors = {}
33
+ hf_safetensors_tensors = {}
34
+
35
+ for name, tensor in instanttensor_weights_iterator(safetensors, True):
36
+ # Copy the tensor immediately as it is a reference to the internal
37
+ # buffer of instanttensor.
38
+ instanttensor_tensors[name] = tensor.to("cpu")
39
+
40
+ for name, tensor in safetensors_weights_iterator(safetensors, True):
41
+ hf_safetensors_tensors[name] = tensor
42
+
43
+ assert len(instanttensor_tensors) == len(hf_safetensors_tensors)
44
+
45
+ for name, instanttensor_tensor in instanttensor_tensors.items():
46
+ assert instanttensor_tensor.dtype == hf_safetensors_tensors[name].dtype
47
+ assert instanttensor_tensor.shape == hf_safetensors_tensors[name].shape
48
+ assert torch.all(instanttensor_tensor.eq(hf_safetensors_tensors[name]))
49
+
50
+
51
+ if __name__ == "__main__":
52
+ test_instanttensor_model_loader()
asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/__init__.py ADDED
File without changes
asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/conftest.py ADDED
@@ -0,0 +1,35 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ from vllm.utils.network_utils import get_distributed_init_method, get_ip, get_open_port
5
+ from vllm.v1.executor import UniProcExecutor
6
+ from vllm.v1.worker.worker_base import WorkerWrapperBase
7
+
8
+
9
+ # This is a dummy executor for patching in test_runai_model_streamer_s3.py.
10
+ # We cannot use vllm_runner fixture here, because it spawns worker process.
11
+ # The worker process reimports the patched entities, and the patch is not applied.
12
+ class RunaiDummyExecutor(UniProcExecutor):
13
+ def _init_executor(self) -> None:
14
+ distributed_init_method = get_distributed_init_method(get_ip(), get_open_port())
15
+
16
+ local_rank = 0
17
+ rank = 0
18
+ is_driver_worker = True
19
+
20
+ device_info = self.vllm_config.device_config.device.__str__().split(":")
21
+ if len(device_info) > 1:
22
+ local_rank = int(device_info[1])
23
+
24
+ worker_rpc_kwargs = dict(
25
+ vllm_config=self.vllm_config,
26
+ local_rank=local_rank,
27
+ rank=rank,
28
+ distributed_init_method=distributed_init_method,
29
+ is_driver_worker=is_driver_worker,
30
+ )
31
+
32
+ self.driver_worker = WorkerWrapperBase()
33
+
34
+ self.collective_rpc("init_worker", args=([worker_rpc_kwargs],))
35
+ self.collective_rpc("init_device")
asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_loader.py ADDED
@@ -0,0 +1,124 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import os
5
+ import types
6
+ from unittest.mock import patch
7
+
8
+ import pytest
9
+
10
+ from vllm import SamplingParams
11
+ from vllm.config.load import LoadConfig
12
+ from vllm.model_executor.model_loader import get_model_loader
13
+ from vllm.model_executor.model_loader import runai_streamer_loader as rsl
14
+
15
+ load_format = "runai_streamer"
16
+ test_model = "openai-community/gpt2"
17
+ # TODO(amacaskill): Replace with a GKE owned GCS bucket.
18
+ test_gcs_model = "gs://vertex-model-garden-public-us/codegemma/codegemma-2b/"
19
+
20
+ prompts = [
21
+ "Hello, my name is",
22
+ "The president of the United States is",
23
+ "The capital of France is",
24
+ "The future of AI is",
25
+ ]
26
+ # Create a sampling params object.
27
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
28
+
29
+
30
+ def get_runai_model_loader():
31
+ load_config = LoadConfig(load_format=load_format)
32
+ return get_model_loader(load_config)
33
+
34
+
35
+ def test_get_model_loader_with_runai_flag():
36
+ model_loader = get_runai_model_loader()
37
+ assert model_loader.__class__.__name__ == "RunaiModelStreamerLoader"
38
+
39
+
40
+ def test_runai_model_loader_download_files(vllm_runner):
41
+ with vllm_runner(test_model, load_format=load_format) as llm:
42
+ deserialized_outputs = llm.generate(prompts, sampling_params)
43
+ assert deserialized_outputs
44
+
45
+
46
+ @pytest.mark.skip(
47
+ reason="Temporarily disabled due to GCS access issues. "
48
+ "TODO: Re-enable this test once the underlying issue is resolved."
49
+ )
50
+ def test_runai_model_loader_download_files_gcs(
51
+ vllm_runner, monkeypatch: pytest.MonkeyPatch
52
+ ):
53
+ monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "fake-project")
54
+ monkeypatch.setenv("RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS", "true")
55
+ monkeypatch.setenv(
56
+ "CLOUD_STORAGE_EMULATOR_ENDPOINT", "https://storage.googleapis.com"
57
+ )
58
+ with vllm_runner(test_gcs_model, load_format=load_format) as llm:
59
+ deserialized_outputs = llm.generate(prompts, sampling_params)
60
+ assert deserialized_outputs
61
+
62
+
63
+ def test_runai_passes_revision_by_name():
64
+ # revision must reach download_safetensors_index_file_from_hf as the
65
+ # ``revision`` keyword, not the positional ``subfolder`` slot.
66
+ fake_self = types.SimpleNamespace(
67
+ load_config=types.SimpleNamespace(download_dir="/cache", ignore_patterns=[])
68
+ )
69
+ with (
70
+ patch.object(rsl, "is_runai_obj_uri", return_value=False),
71
+ patch.object(rsl, "download_weights_from_hf", return_value="/folder"),
72
+ patch.object(
73
+ rsl, "list_safetensors", return_value=["/folder/model.safetensors"]
74
+ ),
75
+ patch.object(rsl, "download_safetensors_index_file_from_hf") as mock_idx,
76
+ ):
77
+ rsl.RunaiModelStreamerLoader._prepare_weights(fake_self, "org/model", "myrev")
78
+
79
+ mock_idx.assert_called_once()
80
+ assert mock_idx.call_args.kwargs.get("revision") == "myrev"
81
+ assert "myrev" not in mock_idx.call_args.args
82
+
83
+
84
+ def _runai_loader(extra):
85
+ return rsl.RunaiModelStreamerLoader(
86
+ LoadConfig(load_format="runai_streamer", model_loader_extra_config=extra)
87
+ )
88
+
89
+
90
+ @pytest.mark.parametrize(
91
+ "extra, match",
92
+ [
93
+ ({"typo_key": 1}, "Unexpected extra config"),
94
+ ({"distributed": "yes"}, "distributed must be a bool"),
95
+ ({"concurrency": "16"}, "concurrency must be a positive integer"),
96
+ ({"concurrency": -1}, "concurrency must be a positive integer"),
97
+ ],
98
+ )
99
+ def test_runai_rejects_invalid_extra_config(extra, match):
100
+ # The loader used to silently drop unknown keys / wrong types / negatives.
101
+ with pytest.raises(ValueError, match=match):
102
+ _runai_loader(extra)
103
+
104
+
105
+ def test_runai_accepts_valid_extra_config():
106
+ with patch.dict(os.environ, {}, clear=False):
107
+ os.environ.pop("RUNAI_STREAMER_CONCURRENCY", None)
108
+ os.environ.pop("RUNAI_STREAMER_MEMORY_LIMIT", None)
109
+ loader = _runai_loader(
110
+ {"distributed": True, "concurrency": 16, "memory_limit": 1024}
111
+ )
112
+ assert loader._is_distributed is True
113
+ assert os.environ["RUNAI_STREAMER_CONCURRENCY"] == "16"
114
+ assert os.environ["RUNAI_STREAMER_MEMORY_LIMIT"] == "1024"
115
+
116
+
117
+ def test_runai_invalid_extra_config_leaves_environ_untouched():
118
+ # A later invalid key must not leave an earlier valid key applied to
119
+ # os.environ (all values are validated before any global mutation).
120
+ with patch.dict(os.environ, {}, clear=False):
121
+ os.environ.pop("RUNAI_STREAMER_CONCURRENCY", None)
122
+ with pytest.raises(ValueError, match="memory_limit must be a positive integer"):
123
+ _runai_loader({"concurrency": 16, "memory_limit": -5})
124
+ assert "RUNAI_STREAMER_CONCURRENCY" not in os.environ
asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py ADDED
@@ -0,0 +1,52 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ from pathlib import Path
5
+
6
+ from huggingface_hub import snapshot_download
7
+ from runai_model_streamer.safetensors_streamer.streamer_mock import StreamerPatcher
8
+
9
+ from vllm.engine.arg_utils import EngineArgs
10
+
11
+ from .conftest import RunaiDummyExecutor
12
+
13
+ load_format = "runai_streamer"
14
+ test_model = "openai-community/gpt2"
15
+
16
+
17
+ def test_runai_model_loader_download_files_s3_mocked_with_patch(
18
+ vllm_runner,
19
+ tmp_path: Path,
20
+ monkeypatch,
21
+ ):
22
+ patcher = StreamerPatcher(str(tmp_path))
23
+
24
+ test_mock_s3_model = "s3://my-mock-bucket/gpt2/"
25
+
26
+ # Download model from HF
27
+ mock_model_dir = f"{tmp_path}/gpt2"
28
+ snapshot_download(repo_id=test_model, local_dir=mock_model_dir)
29
+
30
+ monkeypatch.setattr(
31
+ "vllm.transformers_utils.runai_utils.runai_list_safetensors",
32
+ patcher.shim_list_safetensors,
33
+ )
34
+ monkeypatch.setattr(
35
+ "vllm.transformers_utils.runai_utils.runai_pull_files",
36
+ patcher.shim_pull_files,
37
+ )
38
+ monkeypatch.setattr(
39
+ "vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer",
40
+ patcher.create_mock_streamer,
41
+ )
42
+
43
+ engine_args = EngineArgs(
44
+ model=test_mock_s3_model,
45
+ load_format=load_format,
46
+ tensor_parallel_size=1,
47
+ )
48
+
49
+ vllm_config = engine_args.create_engine_config()
50
+
51
+ executor = RunaiDummyExecutor(vllm_config)
52
+ executor.driver_worker.load_model()
asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_utils.py ADDED
@@ -0,0 +1,60 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import glob
5
+ import hashlib
6
+ import os
7
+ import tempfile
8
+
9
+ import huggingface_hub.constants
10
+
11
+ from vllm.model_executor.model_loader.weight_utils import download_weights_from_hf
12
+ from vllm.transformers_utils.runai_utils import (
13
+ ObjectStorageModel,
14
+ is_runai_obj_uri,
15
+ list_safetensors,
16
+ )
17
+
18
+
19
+ def test_is_runai_obj_uri():
20
+ assert is_runai_obj_uri("gs://some-gcs-bucket/path")
21
+ assert is_runai_obj_uri("s3://some-s3-bucket/path")
22
+ assert is_runai_obj_uri("az://some-azure-container/path")
23
+ assert not is_runai_obj_uri("nfs://some-nfs-path")
24
+
25
+
26
+ def test_runai_list_safetensors_local():
27
+ with tempfile.TemporaryDirectory() as tmpdir:
28
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
29
+ download_weights_from_hf(
30
+ "openai-community/gpt2",
31
+ allow_patterns=["*.safetensors", "*.json"],
32
+ cache_dir=tmpdir,
33
+ )
34
+ safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
35
+ assert len(safetensors) > 0
36
+ parentdir = [os.path.dirname(safetensor) for safetensor in safetensors][0]
37
+ files = list_safetensors(parentdir)
38
+ assert len(safetensors) == len(files)
39
+
40
+
41
+ def test_runai_pull_files_gcs(monkeypatch):
42
+ monkeypatch.setenv("RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS", "true")
43
+ # Bypass default project lookup by setting GOOGLE_CLOUD_PROJECT
44
+ monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "fake-project")
45
+ filename = "LT08_L1GT_074061_20130309_20170505_01_T2_MTL.txt"
46
+ gcs_bucket = "gs://gcp-public-data-landsat/LT08/01/074/061/LT08_L1GT_074061_20130309_20170505_01_T2/"
47
+ gcs_url = f"{gcs_bucket}/{filename}"
48
+ model = ObjectStorageModel(gcs_url)
49
+ model.pull_files(gcs_bucket, allow_pattern=[f"*{filename}"])
50
+ # To re-generate / change URLs:
51
+ # gsutil ls -L gs://<gcs-url> | grep "Hash (md5)" | tr -d ' ' \
52
+ # | cut -d":" -f2 | base64 -d | xxd -p
53
+ expected_checksum = "f60dea775da1392434275b311b31a431"
54
+ hasher = hashlib.new("md5")
55
+ with open(os.path.join(model.dir, filename), "rb") as f:
56
+ # Read the file in chunks to handle large files efficiently
57
+ for chunk in iter(lambda: f.read(4096), b""):
58
+ hasher.update(chunk)
59
+ actual_checksum = hasher.hexdigest()
60
+ assert actual_checksum == expected_checksum
asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_weight_utils.py ADDED
@@ -0,0 +1,66 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import glob
5
+ import tempfile
6
+
7
+ import huggingface_hub.constants
8
+ import torch
9
+ from safetensors.torch import save_file
10
+
11
+ from vllm.model_executor.model_loader.weight_utils import (
12
+ download_weights_from_hf,
13
+ runai_safetensors_weights_iterator,
14
+ safetensors_weights_iterator,
15
+ )
16
+
17
+
18
+ def test_runai_safetensors_weights_iterator_clones_reused_buffers(
19
+ tmp_path, monkeypatch
20
+ ):
21
+ monkeypatch.setenv("RUNAI_STREAMER_MEMORY_LIMIT", "0")
22
+ weights_file = tmp_path / "model.safetensors"
23
+ expected_tensors = {
24
+ "first": torch.tensor([1.0, 2.0]),
25
+ "second": torch.tensor([3.0, 4.0]),
26
+ }
27
+ save_file(expected_tensors, weights_file)
28
+
29
+ actual_tensors = dict(
30
+ runai_safetensors_weights_iterator([str(weights_file)], False)
31
+ )
32
+
33
+ assert actual_tensors.keys() == expected_tensors.keys()
34
+ assert actual_tensors["first"].data_ptr() != actual_tensors["second"].data_ptr()
35
+ for name, expected_tensor in expected_tensors.items():
36
+ assert torch.equal(actual_tensors[name], expected_tensor)
37
+
38
+
39
+ def test_runai_model_loader():
40
+ with tempfile.TemporaryDirectory() as tmpdir:
41
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
42
+ download_weights_from_hf(
43
+ "openai-community/gpt2", allow_patterns=["*.safetensors"], cache_dir=tmpdir
44
+ )
45
+ safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
46
+ assert len(safetensors) > 0
47
+
48
+ runai_model_streamer_tensors = {}
49
+ hf_safetensors_tensors = {}
50
+
51
+ for name, tensor in runai_safetensors_weights_iterator(safetensors, True):
52
+ runai_model_streamer_tensors[name] = tensor
53
+
54
+ for name, tensor in safetensors_weights_iterator(safetensors, True):
55
+ hf_safetensors_tensors[name] = tensor
56
+
57
+ assert len(runai_model_streamer_tensors) == len(hf_safetensors_tensors)
58
+
59
+ for name, runai_tensor in runai_model_streamer_tensors.items():
60
+ assert runai_tensor.dtype == hf_safetensors_tensors[name].dtype
61
+ assert runai_tensor.shape == hf_safetensors_tensors[name].shape
62
+ assert torch.all(runai_tensor.eq(hf_safetensors_tensors[name]))
63
+
64
+
65
+ if __name__ == "__main__":
66
+ test_runai_model_loader()
asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/__init__.py ADDED
File without changes
asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/conftest.py ADDED
@@ -0,0 +1,92 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ from collections.abc import Callable
4
+
5
+ import pytest
6
+
7
+ from vllm import LLM, EngineArgs
8
+ from vllm.distributed import cleanup_dist_env_and_memory
9
+ from vllm.model_executor.model_loader import tensorizer as tensorizer_mod
10
+ from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
11
+ from vllm.utils.network_utils import get_distributed_init_method, get_ip, get_open_port
12
+ from vllm.v1.executor import UniProcExecutor
13
+ from vllm.v1.worker.worker_base import WorkerWrapperBase
14
+
15
+ MODEL_REF = "facebook/opt-125m"
16
+
17
+
18
+ @pytest.fixture()
19
+ def model_ref():
20
+ return MODEL_REF
21
+
22
+
23
+ @pytest.fixture(autouse=True)
24
+ def allow_insecure_serialization(monkeypatch):
25
+ monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
26
+
27
+
28
+ @pytest.fixture(autouse=True)
29
+ def cleanup():
30
+ cleanup_dist_env_and_memory(shutdown_ray=True)
31
+
32
+
33
+ @pytest.fixture()
34
+ def just_serialize_model_tensors(model_ref, monkeypatch, tmp_path):
35
+ def noop(*args, **kwargs):
36
+ return None
37
+
38
+ args = EngineArgs(model=model_ref)
39
+ tc = TensorizerConfig(tensorizer_uri=f"{tmp_path}/model.tensors")
40
+
41
+ monkeypatch.setattr(tensorizer_mod, "serialize_extra_artifacts", noop)
42
+
43
+ tensorizer_mod.tensorize_vllm_model(args, tc)
44
+ yield tmp_path
45
+
46
+
47
+ @pytest.fixture(autouse=True)
48
+ def tensorizer_config():
49
+ config = TensorizerConfig(tensorizer_uri="vllm")
50
+ return config
51
+
52
+
53
+ @pytest.fixture()
54
+ def model_path(model_ref, tmp_path):
55
+ yield tmp_path / model_ref / "model.tensors"
56
+
57
+
58
+ def assert_from_collective_rpc(engine: LLM, closure: Callable, closure_kwargs: dict):
59
+ res = engine.collective_rpc(method=closure, kwargs=closure_kwargs)
60
+ return all(res)
61
+
62
+
63
+ # This is an object pulled from tests/v1/engine/test_engine_core.py
64
+ # Modified to strip the `load_model` method from its `_init_executor`
65
+ # method. It's purely used as a dummy utility to run methods that test
66
+ # Tensorizer functionality
67
+ class DummyExecutor(UniProcExecutor):
68
+ def _init_executor(self) -> None:
69
+ """Initialize the worker and load the model."""
70
+ self.driver_worker = WorkerWrapperBase(rpc_rank=0)
71
+ distributed_init_method = get_distributed_init_method(get_ip(), get_open_port())
72
+ local_rank = 0
73
+ # set local rank as the device index if specified
74
+ device_info = self.vllm_config.device_config.device.__str__().split(":")
75
+ if len(device_info) > 1:
76
+ local_rank = int(device_info[1])
77
+ rank = 0
78
+ is_driver_worker = True
79
+ kwargs = dict(
80
+ vllm_config=self.vllm_config,
81
+ local_rank=local_rank,
82
+ rank=rank,
83
+ distributed_init_method=distributed_init_method,
84
+ is_driver_worker=is_driver_worker,
85
+ )
86
+ self.mm_receiver_cache = None
87
+ self.collective_rpc("init_worker", args=([kwargs],))
88
+ self.collective_rpc("init_device")
89
+
90
+ def shutdown(self):
91
+ if hasattr(self, "thread_pool"):
92
+ self.thread_pool.shutdown(wait=False)
asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py ADDED
@@ -0,0 +1,560 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import asyncio
5
+ import gc
6
+ import json
7
+ import os
8
+ import pathlib
9
+ import subprocess
10
+ import sys
11
+ from typing import Any
12
+
13
+ import pytest
14
+ import torch
15
+
16
+ import vllm.model_executor.model_loader.tensorizer
17
+ from tests.utils import VLLM_PATH, RemoteOpenAIServer
18
+ from vllm import LLM, SamplingParams
19
+ from vllm.engine.arg_utils import EngineArgs
20
+ from vllm.model_executor.model_loader.tensorizer import (
21
+ TensorizerConfig,
22
+ TensorSerializer,
23
+ is_vllm_tensorized,
24
+ open_stream,
25
+ tensorize_vllm_model,
26
+ )
27
+ from vllm.model_executor.model_loader.tensorizer_loader import (
28
+ BLACKLISTED_TENSORIZER_ARGS,
29
+ )
30
+ from vllm.utils.import_utils import PlaceholderModule
31
+
32
+ from .conftest import DummyExecutor, assert_from_collective_rpc
33
+
34
+ try:
35
+ import tensorizer
36
+ from tensorizer import EncryptionParams
37
+ except ImportError:
38
+ tensorizer = PlaceholderModule("tensorizer") # type: ignore[assignment]
39
+ EncryptionParams = tensorizer.placeholder_attr("EncryptionParams")
40
+
41
+
42
+ class TensorizerCaughtError(Exception):
43
+ pass
44
+
45
+
46
+ EXAMPLES_PATH = VLLM_PATH / "examples"
47
+
48
+ pytest_plugins = ("pytest_asyncio",)
49
+
50
+ prompts = [
51
+ "Hello, my name is",
52
+ "The president of the United States is",
53
+ "The capital of France is",
54
+ "The future of AI is",
55
+ ]
56
+ # Create a sampling params object.
57
+ sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
58
+
59
+
60
+ def patch_init_and_catch_error(self, obj, method_name, expected_error: type[Exception]):
61
+ original = getattr(obj, method_name, None)
62
+ if original is None:
63
+ raise ValueError("Method '{}' not found.".format(method_name))
64
+
65
+ def wrapper(*args, **kwargs):
66
+ try:
67
+ return original(*args, **kwargs)
68
+ except expected_error as err:
69
+ raise TensorizerCaughtError from err
70
+
71
+ setattr(obj, method_name, wrapper)
72
+
73
+ self.load_model()
74
+
75
+
76
+ def assert_specific_tensorizer_error_is_raised(
77
+ executor,
78
+ obj: Any,
79
+ method_name: str,
80
+ expected_error: type[Exception],
81
+ ):
82
+ with pytest.raises(TensorizerCaughtError):
83
+ executor.collective_rpc(
84
+ patch_init_and_catch_error,
85
+ args=(
86
+ obj,
87
+ method_name,
88
+ expected_error,
89
+ ),
90
+ )
91
+
92
+
93
+ def is_curl_installed():
94
+ try:
95
+ subprocess.check_call(["curl", "--version"])
96
+ return True
97
+ except (subprocess.CalledProcessError, FileNotFoundError):
98
+ return False
99
+
100
+
101
+ def write_keyfile(keyfile_path: str):
102
+ encryption_params = EncryptionParams.random()
103
+ pathlib.Path(keyfile_path).parent.mkdir(parents=True, exist_ok=True)
104
+ with open(keyfile_path, "wb") as f:
105
+ f.write(encryption_params.key)
106
+
107
+
108
+ @pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
109
+ def test_deserialized_encrypted_vllm_model_has_same_outputs(
110
+ model_ref, vllm_runner, tmp_path, model_path
111
+ ):
112
+ args = EngineArgs(model=model_ref)
113
+ with vllm_runner(model_ref) as vllm_model:
114
+ key_path = tmp_path / model_ref / "model.key"
115
+ write_keyfile(key_path)
116
+
117
+ outputs = vllm_model.generate(prompts, sampling_params)
118
+
119
+ config_for_serializing = TensorizerConfig(
120
+ tensorizer_uri=str(model_path), encryption_keyfile=str(key_path)
121
+ )
122
+
123
+ tensorize_vllm_model(args, config_for_serializing)
124
+
125
+ config_for_deserializing = TensorizerConfig(
126
+ tensorizer_uri=str(model_path), encryption_keyfile=str(key_path)
127
+ )
128
+
129
+ with vllm_runner(
130
+ model_ref,
131
+ load_format="tensorizer",
132
+ model_loader_extra_config=config_for_deserializing,
133
+ ) as loaded_vllm_model: # noqa: E501
134
+ deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
135
+ # noqa: E501
136
+
137
+ assert outputs == deserialized_outputs
138
+
139
+
140
+ def test_deserialized_hf_model_has_same_outputs(
141
+ hf_runner, vllm_runner, tmp_path, model_ref, model_path
142
+ ):
143
+ with hf_runner(model_ref) as hf_model:
144
+ max_tokens = 50
145
+ outputs = hf_model.generate_greedy(prompts, max_tokens=max_tokens)
146
+ with open_stream(model_path, "wb+") as stream:
147
+ serializer = TensorSerializer(stream)
148
+ serializer.write_module(hf_model.model)
149
+
150
+ with vllm_runner(
151
+ model_ref,
152
+ load_format="tensorizer",
153
+ model_loader_extra_config=TensorizerConfig(
154
+ tensorizer_uri=str(model_path),
155
+ num_readers=1,
156
+ ),
157
+ ) as loaded_hf_model:
158
+ deserialized_outputs = loaded_hf_model.generate_greedy(
159
+ prompts, max_tokens=max_tokens
160
+ )
161
+
162
+ assert outputs == deserialized_outputs
163
+
164
+
165
+ def test_load_without_tensorizer_load_format(vllm_runner, capfd, model_ref):
166
+ model = None
167
+ try:
168
+ model = vllm_runner(
169
+ model_ref, model_loader_extra_config=TensorizerConfig(tensorizer_uri="test")
170
+ )
171
+ pytest.fail("Expected RuntimeError for extra config keys")
172
+ except RuntimeError:
173
+ out, err = capfd.readouterr()
174
+ combined_output = out + err
175
+ assert (
176
+ "ValueError: Unexpected extra config keys for load format auto"
177
+ ) in combined_output
178
+ finally:
179
+ del model
180
+ gc.collect()
181
+ torch.accelerator.empty_cache()
182
+
183
+
184
+ def test_raise_value_error_on_invalid_load_format(vllm_runner, capfd, model_ref):
185
+ model = None
186
+ try:
187
+ model = vllm_runner(
188
+ model_ref,
189
+ load_format="safetensors",
190
+ model_loader_extra_config=TensorizerConfig(tensorizer_uri="test"),
191
+ )
192
+ pytest.fail("Expected RuntimeError for extra config keys")
193
+ except RuntimeError:
194
+ out, err = capfd.readouterr()
195
+
196
+ combined_output = out + err
197
+ assert (
198
+ "ValueError: Unexpected extra config keys for load format safetensors"
199
+ ) in combined_output
200
+ finally:
201
+ del model
202
+ gc.collect()
203
+ torch.accelerator.empty_cache()
204
+
205
+
206
+ @pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Requires 2 GPUs")
207
+ def test_tensorizer_with_tp_path_without_template(vllm_runner, capfd):
208
+ try:
209
+ model_ref = "EleutherAI/pythia-1.4b"
210
+ tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
211
+
212
+ vllm_runner(
213
+ model_ref,
214
+ load_format="tensorizer",
215
+ model_loader_extra_config=TensorizerConfig(
216
+ tensorizer_uri=tensorized_path,
217
+ num_readers=1,
218
+ s3_endpoint="object.ord1.coreweave.com",
219
+ ),
220
+ tensor_parallel_size=2,
221
+ disable_custom_all_reduce=True,
222
+ )
223
+ except RuntimeError:
224
+ out, err = capfd.readouterr()
225
+ combined_output = out + err
226
+ assert (
227
+ "ValueError: For a sharded model, tensorizer_uri "
228
+ "should include a string format template like '%04d' "
229
+ "to be formatted with the rank "
230
+ "of the shard"
231
+ ) in combined_output
232
+
233
+
234
+ @pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Requires 2 GPUs")
235
+ def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(
236
+ vllm_runner, tmp_path
237
+ ):
238
+ model_ref = "EleutherAI/pythia-1.4b"
239
+ # record outputs from un-sharded un-tensorized model
240
+ with vllm_runner(
241
+ model_ref,
242
+ disable_custom_all_reduce=True,
243
+ enforce_eager=True,
244
+ ) as base_model:
245
+ outputs = base_model.generate(prompts, sampling_params)
246
+
247
+ # load model with two shards and serialize with encryption
248
+ model_path = str(tmp_path / model_ref / "model-%02d.tensors")
249
+ key_path = tmp_path / (model_ref + ".key")
250
+
251
+ tensorizer_config = TensorizerConfig(
252
+ tensorizer_uri=model_path,
253
+ encryption_keyfile=str(key_path),
254
+ )
255
+
256
+ tensorize_vllm_model(
257
+ engine_args=EngineArgs(
258
+ model=model_ref,
259
+ tensor_parallel_size=2,
260
+ disable_custom_all_reduce=True,
261
+ enforce_eager=True,
262
+ ),
263
+ tensorizer_config=tensorizer_config,
264
+ )
265
+ assert os.path.isfile(model_path % 0), "Serialization subprocess failed"
266
+ assert os.path.isfile(model_path % 1), "Serialization subprocess failed"
267
+
268
+ with vllm_runner(
269
+ model_ref,
270
+ tensor_parallel_size=2,
271
+ load_format="tensorizer",
272
+ disable_custom_all_reduce=True,
273
+ enforce_eager=True,
274
+ model_loader_extra_config=tensorizer_config,
275
+ ) as loaded_vllm_model:
276
+ deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
277
+
278
+ assert outputs == deserialized_outputs
279
+
280
+
281
+ @pytest.mark.flaky(reruns=3)
282
+ def test_vllm_tensorized_model_has_same_outputs(
283
+ model_ref, vllm_runner, tmp_path, model_path
284
+ ):
285
+ gc.collect()
286
+ torch.accelerator.empty_cache()
287
+ config = TensorizerConfig(tensorizer_uri=str(model_path))
288
+ args = EngineArgs(model=model_ref)
289
+
290
+ with vllm_runner(model_ref) as vllm_model:
291
+ outputs = vllm_model.generate(prompts, sampling_params)
292
+
293
+ tensorize_vllm_model(args, config)
294
+ assert is_vllm_tensorized(config)
295
+
296
+ with vllm_runner(
297
+ model_ref, load_format="tensorizer", model_loader_extra_config=config
298
+ ) as loaded_vllm_model:
299
+ deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
300
+ # noqa: E501
301
+
302
+ assert outputs == deserialized_outputs
303
+
304
+
305
+ def test_load_with_just_model_tensors(just_serialize_model_tensors, model_ref):
306
+ # For backwards compatibility, ensure Tensorizer can be still be loaded
307
+ # for inference by passing the model reference name, not a local/S3 dir,
308
+ # and the location of the model tensors
309
+
310
+ model_dir = just_serialize_model_tensors
311
+
312
+ extra_config = {"tensorizer_uri": f"{model_dir}/model.tensors"}
313
+
314
+ ## Start OpenAI API server
315
+ args = [
316
+ "--load-format",
317
+ "tensorizer",
318
+ "--model-loader-extra-config",
319
+ json.dumps(extra_config),
320
+ ]
321
+
322
+ with RemoteOpenAIServer(model_ref, args):
323
+ # This test only concerns itself with being able to load the model
324
+ # and successfully initialize the server
325
+ pass
326
+
327
+
328
+ def test_assert_serialization_kwargs_passed_to_tensor_serializer(tmp_path):
329
+ serialization_params = {
330
+ "limit_cpu_concurrency": 2,
331
+ }
332
+ model_ref = "facebook/opt-125m"
333
+ model_path = tmp_path / (model_ref + ".tensors")
334
+ config = TensorizerConfig(
335
+ tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
336
+ )
337
+ llm = LLM(
338
+ model=model_ref,
339
+ )
340
+
341
+ def serialization_test(self, *args, **kwargs):
342
+ # This is performed in the ephemeral worker process, so monkey-patching
343
+ # will actually work, and cleanup is guaranteed so don't
344
+ # need to reset things
345
+
346
+ original_dict = serialization_params
347
+ to_compare = {}
348
+
349
+ original = tensorizer.serialization.TensorSerializer.__init__
350
+
351
+ def tensorizer_serializer_wrapper(self, *args, **kwargs):
352
+ nonlocal to_compare
353
+ to_compare = kwargs.copy()
354
+ return original(self, *args, **kwargs)
355
+
356
+ tensorizer.serialization.TensorSerializer.__init__ = (
357
+ tensorizer_serializer_wrapper
358
+ )
359
+
360
+ tensorizer_config = TensorizerConfig(**kwargs["tensorizer_config"])
361
+ self.save_tensorized_model(
362
+ tensorizer_config=tensorizer_config,
363
+ )
364
+ return to_compare | original_dict == to_compare
365
+
366
+ kwargs = {"tensorizer_config": config.to_serializable()}
367
+
368
+ assert assert_from_collective_rpc(llm, serialization_test, kwargs)
369
+
370
+
371
+ def test_assert_deserialization_kwargs_passed_to_tensor_deserializer(tmp_path, capfd):
372
+ deserialization_kwargs = {
373
+ "num_readers": "bar", # illegal value
374
+ }
375
+
376
+ serialization_params = {
377
+ "limit_cpu_concurrency": 2,
378
+ }
379
+
380
+ model_ref = "facebook/opt-125m"
381
+ model_path = tmp_path / (model_ref + ".tensors")
382
+ config = TensorizerConfig(
383
+ tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
384
+ )
385
+
386
+ args = EngineArgs(model=model_ref)
387
+ tensorize_vllm_model(args, config)
388
+
389
+ loader_tc = TensorizerConfig(
390
+ tensorizer_uri=str(model_path),
391
+ deserialization_kwargs=deserialization_kwargs,
392
+ )
393
+
394
+ engine_args = EngineArgs(
395
+ model="facebook/opt-125m",
396
+ load_format="tensorizer",
397
+ model_loader_extra_config=loader_tc.to_serializable(),
398
+ )
399
+
400
+ vllm_config = engine_args.create_engine_config()
401
+ executor = DummyExecutor(vllm_config)
402
+
403
+ assert_specific_tensorizer_error_is_raised(
404
+ executor,
405
+ tensorizer.serialization.TensorDeserializer,
406
+ "__init__",
407
+ TypeError,
408
+ )
409
+
410
+
411
+ def test_assert_stream_kwargs_passed_to_tensor_deserializer(tmp_path, capfd):
412
+ deserialization_kwargs = {
413
+ "num_readers": 1,
414
+ }
415
+
416
+ serialization_params = {
417
+ "limit_cpu_concurrency": 2,
418
+ }
419
+
420
+ model_ref = "facebook/opt-125m"
421
+ model_path = tmp_path / (model_ref + ".tensors")
422
+ config = TensorizerConfig(
423
+ tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
424
+ )
425
+
426
+ args = EngineArgs(model=model_ref)
427
+ tensorize_vllm_model(args, config)
428
+
429
+ stream_kwargs = {"mode": "foo"}
430
+
431
+ loader_tc = TensorizerConfig(
432
+ tensorizer_uri=str(model_path),
433
+ deserialization_kwargs=deserialization_kwargs,
434
+ stream_kwargs=stream_kwargs,
435
+ )
436
+
437
+ engine_args = EngineArgs(
438
+ model="facebook/opt-125m",
439
+ load_format="tensorizer",
440
+ model_loader_extra_config=loader_tc.to_serializable(),
441
+ )
442
+
443
+ vllm_config = engine_args.create_engine_config()
444
+ executor = DummyExecutor(vllm_config)
445
+
446
+ assert_specific_tensorizer_error_is_raised(
447
+ executor,
448
+ vllm.model_executor.model_loader.tensorizer,
449
+ "open_stream",
450
+ ValueError,
451
+ )
452
+
453
+
454
+ @pytest.mark.asyncio
455
+ async def test_serialize_and_serve_entrypoints(tmp_path):
456
+ model_ref = "facebook/opt-125m"
457
+
458
+ suffix = "test"
459
+ try:
460
+ result = subprocess.run(
461
+ [
462
+ sys.executable,
463
+ f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py",
464
+ "--model",
465
+ model_ref,
466
+ "serialize",
467
+ "--serialized-directory",
468
+ str(tmp_path),
469
+ "--suffix",
470
+ suffix,
471
+ "--serialization-kwargs",
472
+ '{"limit_cpu_concurrency": 4}',
473
+ ],
474
+ check=True,
475
+ capture_output=True,
476
+ text=True,
477
+ )
478
+ except subprocess.CalledProcessError as e:
479
+ print("Tensorizing failed.")
480
+ print("STDOUT:\n", e.stdout)
481
+ print("STDERR:\n", e.stderr)
482
+ raise
483
+
484
+ assert "Successfully serialized" in result.stdout
485
+
486
+ # Next, try to serve with vllm serve
487
+ model_uri = tmp_path / "vllm" / model_ref / suffix / "model.tensors"
488
+
489
+ model_loader_extra_config = {
490
+ "tensorizer_uri": str(model_uri),
491
+ "stream_kwargs": {
492
+ "force_http": False,
493
+ },
494
+ "deserialization_kwargs": {
495
+ "verify_hash": True,
496
+ "num_readers": 8,
497
+ },
498
+ }
499
+
500
+ cmd = [
501
+ "-m",
502
+ "vllm.entrypoints.cli.main",
503
+ "serve",
504
+ "--host",
505
+ "localhost",
506
+ "--load-format",
507
+ "tensorizer",
508
+ model_ref,
509
+ "--model-loader-extra-config",
510
+ json.dumps(model_loader_extra_config, indent=2),
511
+ ]
512
+
513
+ proc = await asyncio.create_subprocess_exec(
514
+ sys.executable,
515
+ *cmd,
516
+ stdout=asyncio.subprocess.PIPE,
517
+ stderr=asyncio.subprocess.STDOUT,
518
+ )
519
+
520
+ assert proc.stdout is not None
521
+ fut = proc.stdout.readuntil(b"Application startup complete.")
522
+
523
+ try:
524
+ await asyncio.wait_for(fut, 180)
525
+ except asyncio.TimeoutError:
526
+ pytest.fail("Server did not start successfully")
527
+ finally:
528
+ proc.terminate()
529
+ await proc.communicate()
530
+
531
+
532
+ @pytest.mark.parametrize("illegal_value", BLACKLISTED_TENSORIZER_ARGS)
533
+ def test_blacklisted_parameter_for_loading(tmp_path, vllm_runner, capfd, illegal_value):
534
+ serialization_params = {
535
+ "limit_cpu_concurrency": 2,
536
+ }
537
+
538
+ model_ref = "facebook/opt-125m"
539
+ model_path = tmp_path / (model_ref + ".tensors")
540
+ config = TensorizerConfig(
541
+ tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
542
+ )
543
+
544
+ args = EngineArgs(model=model_ref)
545
+ tensorize_vllm_model(args, config)
546
+
547
+ loader_tc = {"tensorizer_uri": str(model_path), illegal_value: "foo"}
548
+
549
+ try:
550
+ vllm_runner(
551
+ model_ref,
552
+ load_format="tensorizer",
553
+ model_loader_extra_config=loader_tc,
554
+ )
555
+ except RuntimeError:
556
+ out, err = capfd.readouterr()
557
+ combined_output = out + err
558
+ assert (
559
+ f"ValueError: {illegal_value} is not an allowed Tensorizer argument."
560
+ ) in combined_output
asian_ai/performance/vLLM/tests/model_executor/model_loader/test_ep_weight_filter.py ADDED
@@ -0,0 +1,361 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ """Tests for EP weight filtering during model loading."""
4
+
5
+ import glob
6
+ import tempfile
7
+
8
+ import huggingface_hub.constants
9
+ import pytest
10
+ import torch
11
+
12
+ from vllm.model_executor.model_loader.ep_weight_filter import (
13
+ compute_local_expert_ids,
14
+ parse_expert_id,
15
+ should_skip_weight,
16
+ )
17
+ from vllm.model_executor.model_loader.weight_utils import (
18
+ safetensors_weights_iterator,
19
+ )
20
+
21
+ # ---------------------------------------------------------------------------
22
+ # Unit tests for parse_expert_id
23
+ # ---------------------------------------------------------------------------
24
+
25
+
26
+ class TestParseExpertId:
27
+ def test_routed_expert(self):
28
+ name = "model.layers.0.mlp.experts.42.gate_proj.weight"
29
+ assert parse_expert_id(name) == 42
30
+
31
+ def test_large_expert_id(self):
32
+ name = "model.layers.60.mlp.experts.383.down_proj.weight"
33
+ assert parse_expert_id(name) == 383
34
+
35
+ def test_shared_expert(self):
36
+ # Shared experts use a different naming convention in most models
37
+ name = "model.layers.0.mlp.shared_experts.gate_proj.weight"
38
+ assert parse_expert_id(name) is None
39
+
40
+ def test_attention_weight(self):
41
+ name = "model.layers.0.self_attn.q_proj.weight"
42
+ assert parse_expert_id(name) is None
43
+
44
+ def test_embedding(self):
45
+ name = "model.embed_tokens.weight"
46
+ assert parse_expert_id(name) is None
47
+
48
+ def test_layernorm(self):
49
+ name = "model.layers.0.input_layernorm.weight"
50
+ assert parse_expert_id(name) is None
51
+
52
+ def test_fused_3d_expert(self):
53
+ # 3D fused-expert tensors (e.g. gpt-oss) have no numeric expert id.
54
+ # They must NOT be filtered — slicing happens later in weight_loader.
55
+ name = "model.layers.0.mlp.experts.gate_proj.weight"
56
+ assert parse_expert_id(name) is None
57
+
58
+ def test_fused_3d_expert_down_proj(self):
59
+ name = "model.layers.10.mlp.experts.down_proj.weight"
60
+ assert parse_expert_id(name) is None
61
+
62
+ def test_expert_scale(self):
63
+ # NVFP4 quantized models have scale tensors for experts
64
+ name = "model.layers.5.mlp.experts.100.gate_proj.weight_scale"
65
+ assert parse_expert_id(name) == 100
66
+
67
+ def test_expert_zero_id(self):
68
+ name = "model.layers.0.mlp.experts.0.up_proj.weight"
69
+ assert parse_expert_id(name) == 0
70
+
71
+
72
+ # ---------------------------------------------------------------------------
73
+ # Unit tests for compute_local_expert_ids
74
+ # ---------------------------------------------------------------------------
75
+
76
+
77
+ class TestComputeLocalExpertIds:
78
+ def test_ep_disabled(self):
79
+ assert compute_local_expert_ids(64, ep_size=1, ep_rank=0) is None
80
+
81
+ def test_even_split(self):
82
+ # 64 experts, EP=8 → 8 per rank
83
+ ids = compute_local_expert_ids(64, ep_size=8, ep_rank=0)
84
+ assert ids == set(range(0, 8))
85
+
86
+ ids = compute_local_expert_ids(64, ep_size=8, ep_rank=7)
87
+ assert ids == set(range(56, 64))
88
+
89
+ def test_uneven_split(self):
90
+ # 10 experts, EP=3 → ranks get 4, 3, 3
91
+ ids_0 = compute_local_expert_ids(10, ep_size=3, ep_rank=0)
92
+ ids_1 = compute_local_expert_ids(10, ep_size=3, ep_rank=1)
93
+ ids_2 = compute_local_expert_ids(10, ep_size=3, ep_rank=2)
94
+
95
+ assert len(ids_0) == 4
96
+ assert len(ids_1) == 3
97
+ assert len(ids_2) == 3
98
+ # All experts covered, no overlap
99
+ assert ids_0 | ids_1 | ids_2 == set(range(10))
100
+ assert ids_0.isdisjoint(ids_1)
101
+ assert ids_1.isdisjoint(ids_2)
102
+
103
+ def test_384_experts_ep8(self):
104
+ # Kimi-K2.5 config: 384 experts, EP=8
105
+ for rank in range(8):
106
+ ids = compute_local_expert_ids(384, ep_size=8, ep_rank=rank)
107
+ assert len(ids) == 48
108
+
109
+ # All experts covered
110
+ all_ids = set()
111
+ for rank in range(8):
112
+ ids = compute_local_expert_ids(384, ep_size=8, ep_rank=rank)
113
+ all_ids |= ids
114
+ assert all_ids == set(range(384))
115
+
116
+ def test_384_experts_ep16(self):
117
+ for rank in range(16):
118
+ ids = compute_local_expert_ids(384, ep_size=16, ep_rank=rank)
119
+ assert len(ids) == 24
120
+
121
+ def test_384_experts_ep24(self):
122
+ # 384 / 24 = 16 exactly
123
+ for rank in range(24):
124
+ ids = compute_local_expert_ids(384, ep_size=24, ep_rank=rank)
125
+ assert len(ids) == 16
126
+
127
+ # round_robin placement tests
128
+
129
+ def test_round_robin_basic(self):
130
+ # 8 experts, EP=2: rank 0 → {0,2,4,6}, rank 1 → {1,3,5,7}
131
+ rr = "round_robin"
132
+ ids_0 = compute_local_expert_ids(8, 2, 0, placement=rr)
133
+ ids_1 = compute_local_expert_ids(8, 2, 1, placement=rr)
134
+ assert ids_0 == {0, 2, 4, 6}
135
+ assert ids_1 == {1, 3, 5, 7}
136
+
137
+ def test_round_robin_full_coverage(self):
138
+ # 384 experts, EP=8: all experts covered, no overlap
139
+ rr = "round_robin"
140
+ all_ids: set[int] = set()
141
+ for rank in range(8):
142
+ ids = compute_local_expert_ids(384, 8, rank, placement=rr)
143
+ assert ids is not None and len(ids) == 48
144
+ assert all_ids.isdisjoint(ids)
145
+ all_ids |= ids
146
+ assert all_ids == set(range(384))
147
+
148
+ def test_round_robin_uneven(self):
149
+ # 10 experts, EP=3: rank 0→{0,3,6,9}, rank 1→{1,4,7}, rank 2→{2,5,8}
150
+ rr = "round_robin"
151
+ ids_0 = compute_local_expert_ids(10, 3, 0, placement=rr)
152
+ ids_1 = compute_local_expert_ids(10, 3, 1, placement=rr)
153
+ ids_2 = compute_local_expert_ids(10, 3, 2, placement=rr)
154
+ assert ids_0 == {0, 3, 6, 9}
155
+ assert ids_1 == {1, 4, 7}
156
+ assert ids_2 == {2, 5, 8}
157
+ assert ids_0 | ids_1 | ids_2 == set(range(10))
158
+
159
+
160
+ # ---------------------------------------------------------------------------
161
+ # Unit tests for should_skip_weight
162
+ # ---------------------------------------------------------------------------
163
+
164
+
165
+ class TestShouldSkipWeight:
166
+ def setup_method(self):
167
+ # Simulate EP=8, rank=0 → experts 0-47
168
+ self.local_ids = compute_local_expert_ids(384, ep_size=8, ep_rank=0)
169
+
170
+ def test_no_filter(self):
171
+ assert not should_skip_weight("anything", None)
172
+
173
+ def test_dense_not_skipped(self):
174
+ assert not should_skip_weight(
175
+ "model.layers.0.self_attn.q_proj.weight", self.local_ids
176
+ )
177
+
178
+ def test_local_expert_not_skipped(self):
179
+ assert not should_skip_weight(
180
+ "model.layers.0.mlp.experts.10.gate_proj.weight", self.local_ids
181
+ )
182
+
183
+ def test_remote_expert_skipped(self):
184
+ assert should_skip_weight(
185
+ "model.layers.0.mlp.experts.200.gate_proj.weight", self.local_ids
186
+ )
187
+
188
+ def test_boundary_expert(self):
189
+ # Expert 47 is local (last one), 48 is not
190
+ assert not should_skip_weight(
191
+ "model.layers.0.mlp.experts.47.gate_proj.weight", self.local_ids
192
+ )
193
+ assert should_skip_weight(
194
+ "model.layers.0.mlp.experts.48.gate_proj.weight", self.local_ids
195
+ )
196
+
197
+ def test_shared_expert_not_skipped(self):
198
+ assert not should_skip_weight(
199
+ "model.layers.0.mlp.shared_experts.gate_proj.weight", self.local_ids
200
+ )
201
+
202
+ def test_embedding_not_skipped(self):
203
+ assert not should_skip_weight("model.embed_tokens.weight", self.local_ids)
204
+
205
+ def test_fused_3d_expert_not_skipped(self):
206
+ # 3D fused-expert tensors (gpt-oss style) have no numeric id.
207
+ # Must not be skipped — weight_loader handles slicing later.
208
+ assert not should_skip_weight(
209
+ "model.layers.0.mlp.experts.gate_proj.weight", self.local_ids
210
+ )
211
+
212
+
213
+ # ---------------------------------------------------------------------------
214
+ # Integration test: safetensors_weights_iterator with EP filtering
215
+ # ---------------------------------------------------------------------------
216
+
217
+
218
+ class TestSafetensorsWeightsIteratorWithEpFilter:
219
+ """Verify that EP filtering produces a strict subset of unfiltered loading
220
+ and that all expected dense + local expert weights are present."""
221
+
222
+ @pytest.fixture(scope="class")
223
+ def gpt2_files(self):
224
+ """Download GPT-2 safetensors to a temp dir (shared across class)."""
225
+ with tempfile.TemporaryDirectory() as tmpdir:
226
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
227
+ from vllm.model_executor.model_loader.weight_utils import (
228
+ download_weights_from_hf,
229
+ )
230
+
231
+ download_weights_from_hf(
232
+ "openai-community/gpt2",
233
+ allow_patterns=["*.safetensors"],
234
+ cache_dir=tmpdir,
235
+ )
236
+ files = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
237
+ assert len(files) > 0
238
+ yield files
239
+
240
+ def test_no_filter_returns_all(self, gpt2_files):
241
+ """With local_expert_ids=None, all weights are returned (no MoE)."""
242
+ all_weights = dict(safetensors_weights_iterator(gpt2_files, False))
243
+ filtered_weights = dict(
244
+ safetensors_weights_iterator(gpt2_files, False, local_expert_ids=None)
245
+ )
246
+ assert set(all_weights.keys()) == set(filtered_weights.keys())
247
+
248
+ def test_empty_filter_skips_experts_only(self, gpt2_files):
249
+ """GPT-2 has no expert weights, so even an empty local_expert_ids
250
+ set should return all weights (all are dense)."""
251
+ all_weights = dict(safetensors_weights_iterator(gpt2_files, False))
252
+ filtered_weights = dict(
253
+ safetensors_weights_iterator(gpt2_files, False, local_expert_ids=set())
254
+ )
255
+ # GPT-2 has no experts, so nothing should be filtered
256
+ assert set(all_weights.keys()) == set(filtered_weights.keys())
257
+
258
+
259
+ class TestEpFilterOnSyntheticMoeWeights:
260
+ """Create synthetic safetensors files with expert-like naming and verify
261
+ that the filter correctly skips non-local experts."""
262
+
263
+ @pytest.fixture
264
+ def synthetic_moe_files(self, tmp_path):
265
+ """Create synthetic safetensors with expert-patterned tensor names."""
266
+ from safetensors.torch import save_file
267
+
268
+ tensors = {}
269
+ # Dense weights
270
+ tensors["model.embed_tokens.weight"] = torch.randn(100, 64)
271
+ tensors["model.layers.0.self_attn.q_proj.weight"] = torch.randn(64, 64)
272
+ tensors["model.layers.0.input_layernorm.weight"] = torch.randn(64)
273
+ # Expert weights: 8 experts
274
+ for expert_id in range(8):
275
+ tensors[f"model.layers.0.mlp.experts.{expert_id}.gate_proj.weight"] = (
276
+ torch.randn(128, 64)
277
+ )
278
+ tensors[f"model.layers.0.mlp.experts.{expert_id}.up_proj.weight"] = (
279
+ torch.randn(128, 64)
280
+ )
281
+ tensors[f"model.layers.0.mlp.experts.{expert_id}.down_proj.weight"] = (
282
+ torch.randn(64, 128)
283
+ )
284
+ # Shared expert (should never be filtered)
285
+ tensors["model.layers.0.mlp.shared_experts.gate_proj.weight"] = torch.randn(
286
+ 128, 64
287
+ )
288
+
289
+ filepath = str(tmp_path / "model-00001-of-00001.safetensors")
290
+ save_file(tensors, filepath)
291
+ return [filepath], tensors
292
+
293
+ def test_no_filter_returns_all(self, synthetic_moe_files):
294
+ files, expected = synthetic_moe_files
295
+ loaded = dict(safetensors_weights_iterator(files, False))
296
+ assert set(loaded.keys()) == set(expected.keys())
297
+
298
+ def test_ep2_rank0_gets_half_experts(self, synthetic_moe_files):
299
+ files, expected = synthetic_moe_files
300
+ # EP=2, rank=0 → experts 0-3
301
+ local_ids = compute_local_expert_ids(8, ep_size=2, ep_rank=0)
302
+ loaded = dict(
303
+ safetensors_weights_iterator(files, False, local_expert_ids=local_ids)
304
+ )
305
+
306
+ # Should have all dense + shared + experts 0-3 only
307
+ for name in loaded:
308
+ eid = parse_expert_id(name)
309
+ if eid is not None:
310
+ assert eid in local_ids, f"Non-local expert {eid} was loaded"
311
+
312
+ # Check expert count: 4 experts × 3 weights = 12
313
+ expert_names = [n for n in loaded if parse_expert_id(n) is not None]
314
+ assert len(expert_names) == 4 * 3
315
+
316
+ # Check all dense weights present
317
+ assert "model.embed_tokens.weight" in loaded
318
+ assert "model.layers.0.self_attn.q_proj.weight" in loaded
319
+ assert "model.layers.0.input_layernorm.weight" in loaded
320
+ assert "model.layers.0.mlp.shared_experts.gate_proj.weight" in loaded
321
+
322
+ def test_ep2_rank1_gets_other_half(self, synthetic_moe_files):
323
+ files, expected = synthetic_moe_files
324
+ local_ids = compute_local_expert_ids(8, ep_size=2, ep_rank=1)
325
+ loaded = dict(
326
+ safetensors_weights_iterator(files, False, local_expert_ids=local_ids)
327
+ )
328
+
329
+ expert_names = [n for n in loaded if parse_expert_id(n) is not None]
330
+ assert len(expert_names) == 4 * 3
331
+ for name in expert_names:
332
+ assert parse_expert_id(name) in local_ids
333
+
334
+ def test_ep8_each_rank_gets_one_expert(self, synthetic_moe_files):
335
+ files, _ = synthetic_moe_files
336
+ all_expert_names = set()
337
+ for rank in range(8):
338
+ local_ids = compute_local_expert_ids(8, ep_size=8, ep_rank=rank)
339
+ loaded = dict(
340
+ safetensors_weights_iterator(files, False, local_expert_ids=local_ids)
341
+ )
342
+ expert_names = {n for n in loaded if parse_expert_id(n) is not None}
343
+ # 1 expert × 3 weights
344
+ assert len(expert_names) == 3
345
+ all_expert_names |= expert_names
346
+
347
+ # All 8 experts × 3 weights covered across ranks
348
+ assert len(all_expert_names) == 24
349
+
350
+ def test_tensor_values_match(self, synthetic_moe_files):
351
+ """Filtered tensors have identical values to unfiltered ones."""
352
+ files, _ = synthetic_moe_files
353
+ all_weights = dict(safetensors_weights_iterator(files, False))
354
+
355
+ local_ids = compute_local_expert_ids(8, ep_size=2, ep_rank=0)
356
+ filtered = dict(
357
+ safetensors_weights_iterator(files, False, local_expert_ids=local_ids)
358
+ )
359
+
360
+ for name, tensor in filtered.items():
361
+ assert torch.equal(tensor, all_weights[name]), f"Tensor mismatch for {name}"
asian_ai/performance/vLLM/tests/model_executor/model_loader/test_modelexpress_loader.py ADDED
@@ -0,0 +1,133 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import sys
5
+ from types import ModuleType, SimpleNamespace
6
+
7
+ import pytest
8
+ from torch import nn
9
+
10
+ from vllm.config import VllmConfig
11
+ from vllm.config.load import LoadConfig
12
+ from vllm.model_executor.model_loader import get_model_loader
13
+ from vllm.model_executor.model_loader.modelexpress_loader import (
14
+ ModelExpressModelLoader,
15
+ )
16
+
17
+
18
+ class FakeModelexpressLoader:
19
+ calls: list[tuple[str, tuple, dict]] = []
20
+ loaded_model: nn.Module
21
+
22
+ def __init__(self, load_config: LoadConfig):
23
+ self.load_config = load_config
24
+
25
+ def download_model(self, *args, **kwargs):
26
+ self.calls.append(("download_model", args, kwargs))
27
+
28
+ def load_weights(self, *args, **kwargs):
29
+ self.calls.append(("load_weights", args, kwargs))
30
+
31
+ def load_model(self, *args, **kwargs):
32
+ self.calls.append(("load_model", args, kwargs))
33
+ return self.loaded_model
34
+
35
+
36
+ def _install_fake_modelexpress(monkeypatch):
37
+ FakeModelexpressLoader.calls = []
38
+ FakeModelexpressLoader.loaded_model = nn.Module()
39
+
40
+ for name in [
41
+ "modelexpress",
42
+ "modelexpress.engines",
43
+ "modelexpress.engines.vllm",
44
+ ]:
45
+ monkeypatch.setitem(sys.modules, name, ModuleType(name))
46
+
47
+ module = ModuleType("modelexpress.engines.vllm.loader")
48
+ module.__dict__["MxModelLoader"] = FakeModelexpressLoader
49
+ monkeypatch.setitem(sys.modules, module.__name__, module)
50
+
51
+
52
+ def test_modelexpress_load_format_resolves_to_modelexpress_loader(monkeypatch):
53
+ _install_fake_modelexpress(monkeypatch)
54
+
55
+ loader = get_model_loader(LoadConfig(load_format="modelexpress"))
56
+
57
+ assert isinstance(loader, ModelExpressModelLoader)
58
+
59
+
60
+ def test_modelexpress_loader_delegates_to_modelexpress(monkeypatch):
61
+ _install_fake_modelexpress(monkeypatch)
62
+ loader = ModelExpressModelLoader(LoadConfig(load_format="modelexpress"))
63
+ model = nn.Module()
64
+ model_config = SimpleNamespace()
65
+ vllm_config = SimpleNamespace()
66
+
67
+ loader.download_model(model_config)
68
+ loader.load_weights(model, model_config)
69
+ FakeModelexpressLoader.loaded_model.train()
70
+ result = loader.load_model(
71
+ vllm_config=vllm_config,
72
+ model_config=model_config,
73
+ prefix="model",
74
+ )
75
+
76
+ assert result is FakeModelexpressLoader.loaded_model
77
+ assert not result.training
78
+ assert FakeModelexpressLoader.calls == [
79
+ ("download_model", (model_config,), {}),
80
+ ("load_weights", (model, model_config), {}),
81
+ (
82
+ "load_model",
83
+ (),
84
+ {
85
+ "vllm_config": vllm_config,
86
+ "model_config": model_config,
87
+ "prefix": "model",
88
+ },
89
+ ),
90
+ ]
91
+
92
+
93
+ def test_modelexpress_loader_missing_modelexpress_error(monkeypatch):
94
+ import importlib
95
+
96
+ def missing_modelexpress(name):
97
+ raise ModuleNotFoundError(name=name)
98
+
99
+ monkeypatch.setattr(importlib, "import_module", missing_modelexpress)
100
+
101
+ with pytest.raises(ImportError, match="requires the ModelExpress Python package"):
102
+ ModelExpressModelLoader(LoadConfig(load_format="modelexpress"))
103
+
104
+
105
+ def test_modelexpress_loader_preserves_internal_import_errors(monkeypatch):
106
+ import importlib
107
+
108
+ def missing_dependency(name):
109
+ raise ModuleNotFoundError(name="not_modelexpress_dependency")
110
+
111
+ monkeypatch.setattr(importlib, "import_module", missing_dependency)
112
+
113
+ with pytest.raises(ModuleNotFoundError) as exc_info:
114
+ ModelExpressModelLoader(LoadConfig(load_format="modelexpress"))
115
+ assert exc_info.value.name == "not_modelexpress_dependency"
116
+
117
+
118
+ def test_modelexpress_load_format_allows_object_storage_model_weights():
119
+ model_config = SimpleNamespace(
120
+ architecture="UnknownForTest",
121
+ config_updated=False,
122
+ convert_type=None,
123
+ is_hybrid=False,
124
+ model="test-model",
125
+ model_weights="s3://bucket/model",
126
+ )
127
+ vllm_config = object.__new__(VllmConfig)
128
+ vllm_config.model_config = model_config
129
+ vllm_config.load_config = LoadConfig(load_format="modelexpress")
130
+
131
+ vllm_config.try_verify_and_update_config()
132
+
133
+ assert vllm_config.load_config.load_format == "modelexpress"
asian_ai/performance/vLLM/tests/model_executor/model_loader/test_registry.py ADDED
@@ -0,0 +1,90 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import pytest
5
+ from torch import nn
6
+
7
+ from vllm.config import ModelConfig
8
+ from vllm.config.load import LoadConfig
9
+ from vllm.model_executor.model_loader import get_model_loader, register_model_loader
10
+ from vllm.model_executor.model_loader.base_loader import BaseModelLoader
11
+ from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
12
+
13
+
14
+ @register_model_loader("custom_load_format")
15
+ class CustomModelLoader(BaseModelLoader):
16
+ def __init__(self, load_config: LoadConfig) -> None:
17
+ super().__init__(load_config)
18
+
19
+ def download_model(self, model_config: ModelConfig) -> None:
20
+ pass
21
+
22
+ def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
23
+ pass
24
+
25
+
26
+ def test_register_model_loader():
27
+ load_config = LoadConfig(load_format="custom_load_format")
28
+ assert isinstance(get_model_loader(load_config), CustomModelLoader)
29
+
30
+
31
+ def test_invalid_model_loader():
32
+ with pytest.raises(ValueError):
33
+
34
+ @register_model_loader("invalid_load_format")
35
+ class InValidModelLoader:
36
+ pass
37
+
38
+
39
+ def test_default_loader_rejects_zero_num_threads():
40
+ # num_threads=0 used to fail late in ThreadPoolExecutor ("max_workers must be > 0").
41
+ with pytest.raises(ValueError, match="num_threads"):
42
+ DefaultModelLoader(
43
+ LoadConfig(
44
+ model_loader_extra_config={
45
+ "enable_multithread_load": True,
46
+ "num_threads": 0,
47
+ }
48
+ )
49
+ )
50
+
51
+
52
+ def test_default_loader_rejects_multithread_with_non_lazy_strategy():
53
+ # The multi-thread loader ignores safetensors_load_strategy; reject the
54
+ # combination instead of silently dropping the requested strategy.
55
+ with pytest.raises(ValueError, match="does not support"):
56
+ DefaultModelLoader(
57
+ LoadConfig(
58
+ safetensors_load_strategy="torchao",
59
+ model_loader_extra_config={"enable_multithread_load": True},
60
+ )
61
+ )
62
+
63
+
64
+ def test_default_loader_explicit_safetensors_does_not_misread_pt(tmp_path):
65
+ # Explicit safetensors must not fall back to a .pt and open it as safetensors.
66
+ (tmp_path / "model.pt").write_bytes(b"\x00\x00\x00\x00")
67
+ loader = DefaultModelLoader(LoadConfig(load_format="safetensors"))
68
+ with pytest.raises(RuntimeError, match="Cannot find any model weights"):
69
+ loader._prepare_weights(
70
+ str(tmp_path),
71
+ None,
72
+ None,
73
+ fall_back_to_pt=True,
74
+ allow_patterns_overrides=None,
75
+ )
76
+
77
+
78
+ def test_default_loader_hf_still_falls_back_to_pt(tmp_path):
79
+ # Control: load_format="hf" still picks up .pt weights via fallback.
80
+ (tmp_path / "model.pt").write_bytes(b"\x00\x00\x00\x00")
81
+ loader = DefaultModelLoader(LoadConfig(load_format="hf"))
82
+ _, files, use_safetensors = loader._prepare_weights(
83
+ str(tmp_path),
84
+ None,
85
+ None,
86
+ fall_back_to_pt=True,
87
+ allow_patterns_overrides=None,
88
+ )
89
+ assert use_safetensors is False
90
+ assert any(f.endswith("model.pt") for f in files)
asian_ai/performance/vLLM/tests/model_executor/model_loader/test_reload.py ADDED
@@ -0,0 +1,498 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ import gc
4
+ import inspect
5
+ from weakref import WeakKeyDictionary, ref
6
+
7
+ import pytest
8
+ import torch
9
+ from torch.nn.parameter import UninitializedParameter
10
+
11
+ import vllm.model_executor.model_loader.reload.meta as reload_meta
12
+ from vllm.model_executor.layers.linear import QKVParallelLinear
13
+ from vllm.model_executor.model_loader.reload.layerwise import (
14
+ finalize_layerwise_reload,
15
+ initialize_layerwise_reload,
16
+ record_metadata_for_reloading,
17
+ )
18
+ from vllm.model_executor.model_loader.reload.meta import (
19
+ capture_layer_to_meta,
20
+ get_numel_loaded,
21
+ materialize_layer,
22
+ materialize_meta_tensor,
23
+ restore_layer_on_meta,
24
+ to_meta_tensor,
25
+ )
26
+ from vllm.model_executor.model_loader.reload.types import LayerReloadingInfo
27
+ from vllm.model_executor.model_loader.reload.utils import get_layer_tensors
28
+ from vllm.model_executor.model_loader.weight_utils import (
29
+ composed_weight_loader,
30
+ default_weight_loader,
31
+ )
32
+ from vllm.platforms import current_platform
33
+
34
+
35
+ def _fp8_reload_unsupported() -> bool:
36
+ """Whether the FP8 reload/online-quantize tests should be skipped.
37
+
38
+ ``supports_fp8()`` returns True on MI250 (gfx90a) because the general
39
+ quantization paths upcast FP8 weights, but gfx90a has no native FP8 and
40
+ cannot run these reload models, so treat it as unsupported here.
41
+ """
42
+ if not current_platform.supports_fp8():
43
+ return True
44
+ if current_platform.is_rocm():
45
+ from vllm.platforms.rocm import on_gfx90a
46
+
47
+ return on_gfx90a()
48
+ return False
49
+
50
+
51
+ class _AliasedBufferLayer(torch.nn.Module):
52
+ def __init__(self):
53
+ super().__init__()
54
+ weight = torch.arange(6, dtype=torch.float32).reshape(2, 3)
55
+ self.weight = torch.nn.Parameter(weight)
56
+ self.register_buffer(
57
+ "weight_view", self.weight.detach().view(-1), persistent=False
58
+ )
59
+
60
+
61
+ class _ParentAliasedChildBufferLayer(torch.nn.Module):
62
+ def __init__(self):
63
+ super().__init__()
64
+ self.scale = torch.nn.Parameter(torch.ones(1))
65
+ self.conv1d = torch.nn.Linear(3, 2, bias=False)
66
+ self.conv1d.weight.data.copy_(
67
+ torch.arange(6, dtype=torch.float32).reshape(2, 3)
68
+ )
69
+ self.register_buffer(
70
+ "conv_weights", self.conv1d.weight.detach().view(-1), persistent=False
71
+ )
72
+
73
+
74
+ class _AliasedBufferWithUninitializedChildLayer(_AliasedBufferLayer):
75
+ def __init__(self):
76
+ super().__init__()
77
+ self.child = torch.nn.Module()
78
+ self.child.register_parameter(
79
+ "lazy_weight", UninitializedParameter(requires_grad=False)
80
+ )
81
+
82
+
83
+ def test_move_metatensors():
84
+ tensor = torch.empty((1, 2, 3))
85
+ meta_tensor = to_meta_tensor(tensor)
86
+ materialized_tensor = materialize_meta_tensor(meta_tensor)
87
+
88
+ assert meta_tensor.device.type == "meta"
89
+ assert tensor.device == materialized_tensor.device
90
+
91
+ assert tensor.dtype == meta_tensor.dtype == materialized_tensor.dtype
92
+ assert tensor.shape == meta_tensor.shape == materialized_tensor.shape
93
+ assert tensor.__class__ == meta_tensor.__class__ == materialized_tensor.__class__
94
+ assert tensor.__dict__ == meta_tensor.__dict__ == materialized_tensor.__dict__
95
+
96
+
97
+ def test_reload_lifecycle():
98
+ layer = torch.nn.Linear(2, 3)
99
+ info = LayerReloadingInfo(
100
+ restore_metadata=capture_layer_to_meta(layer),
101
+ restore_device=torch.device("cpu"),
102
+ )
103
+
104
+ restore_layer_on_meta(layer, info)
105
+ for name, tensor in get_layer_tensors(layer).items():
106
+ meta_tensor = getattr(layer, name)
107
+ assert tensor.dtype == meta_tensor.dtype
108
+ assert tensor.shape == meta_tensor.shape
109
+ assert tensor.__class__ == meta_tensor.__class__
110
+ assert tensor.__dict__ == meta_tensor.__dict__
111
+
112
+ materialize_layer(layer, info)
113
+ for name, tensor in get_layer_tensors(layer).items():
114
+ materialized_tensor = getattr(layer, name)
115
+ assert tensor.dtype == materialized_tensor.dtype
116
+ assert tensor.shape == materialized_tensor.shape
117
+ assert tensor.__class__ == materialized_tensor.__class__
118
+ assert tensor.__dict__ == materialized_tensor.__dict__
119
+
120
+
121
+ def test_materialize_layer_preserves_non_meta_tensors():
122
+ """Ensure that materialize_layer does not overwrite non meta tensors."""
123
+ layer = torch.nn.Linear(2, 3, bias=True)
124
+
125
+ # Create a non meta bias tensor and meta weight, which can happen with FP8
126
+ bias_values = torch.ones(3)
127
+ layer.bias.data.copy_(bias_values)
128
+ layer.weight = torch.nn.Parameter(layer.weight.data.to("meta"))
129
+
130
+ assert layer.weight.is_meta
131
+ assert not layer.bias.is_meta
132
+
133
+ # materialize the layer weights after the bias is initialized
134
+ info = LayerReloadingInfo(
135
+ restore_metadata=({}, {}),
136
+ restore_device=torch.device("cpu"),
137
+ )
138
+ materialize_layer(layer, info)
139
+
140
+ # Ensure the weight materialized off meta
141
+ assert not layer.weight.is_meta
142
+ assert layer.weight.device.type == "cpu"
143
+
144
+ # Ensure that the bias is (still) not meta and values are unchanged
145
+ assert not layer.bias.is_meta
146
+ assert torch.equal(layer.bias.data, bias_values)
147
+
148
+
149
+ def test_model_cleanup(dist_init, default_vllm_config):
150
+ layer = QKVParallelLinear(2, 3, 4)
151
+ assert layer.weight.weight_loader.__self__ is layer
152
+ info = LayerReloadingInfo(
153
+ restore_metadata=capture_layer_to_meta(layer),
154
+ restore_device=torch.device("cpu"),
155
+ )
156
+
157
+ mock_info_dict: WeakKeyDictionary[torch.nn.Module, LayerReloadingInfo] = (
158
+ WeakKeyDictionary()
159
+ )
160
+ mock_info_dict[layer] = info
161
+ layer_ref = ref(layer)
162
+
163
+ del layer
164
+ gc.collect()
165
+
166
+ assert layer_ref() is None
167
+ assert len(mock_info_dict) == 0
168
+
169
+
170
+ def test_get_numel_loaded():
171
+ param = torch.empty(10, device="meta")
172
+ loaded_weight = torch.empty(10)
173
+
174
+ def complex_weight_loader(param, loaded_weight):
175
+ param[:3] = loaded_weight[:3]
176
+ param[5:8] = loaded_weight[5:8]
177
+ return "value"
178
+
179
+ args = inspect.signature(complex_weight_loader).bind(param, loaded_weight)
180
+ num_loaded, ret = get_numel_loaded(complex_weight_loader, args)
181
+ assert num_loaded == 6
182
+ assert ret == "value"
183
+
184
+
185
+ def test_get_numel_loaded_caps_at_param_size():
186
+ # composed_weight_loader copies into the param twice (the load and the
187
+ # in-place post-load transform), but only param.numel() distinct elements
188
+ # are loaded. get_numel_loaded must not double-count, otherwise a layer's
189
+ # loaded-element total can be reached early and trailing params get dropped.
190
+ param = torch.empty(10)
191
+ loaded_weight = torch.ones(10)
192
+ loader = composed_weight_loader(default_weight_loader, lambda x: x + 1)
193
+
194
+ args = inspect.signature(loader).bind(param, loaded_weight)
195
+ num_loaded, _ = get_numel_loaded(loader, args)
196
+ assert num_loaded == 10
197
+
198
+
199
+ class _ComposedLoaderLayer(torch.nn.Module):
200
+ """Mimics a Mamba2 mixer's equal-numel direct params (A, D, dt_bias).
201
+
202
+ ``A`` uses ``composed_weight_loader`` (an extra in-place transform copy),
203
+ matching ``MambaMixer2`` where ``A`` is loaded as ``-exp(A_log)``.
204
+ """
205
+
206
+ def __init__(self):
207
+ super().__init__()
208
+ self.A = torch.nn.Parameter(torch.empty(4, dtype=torch.float32))
209
+ self.D = torch.nn.Parameter(torch.ones(4))
210
+ self.dt_bias = torch.nn.Parameter(torch.ones(4))
211
+ self.A.weight_loader = composed_weight_loader(
212
+ default_weight_loader, lambda x: -torch.exp(x.float())
213
+ )
214
+ self.D.weight_loader = default_weight_loader
215
+ self.dt_bias.weight_loader = default_weight_loader
216
+
217
+
218
+ def test_layerwise_reload_composed_loader_does_not_drop_params(monkeypatch):
219
+ # Regression test: a composed_weight_loader param (A) used to double-count
220
+ # its elements, finalizing the layer before the trailing param (D) was
221
+ # loaded and leaving it as uninitialized materialized memory.
222
+ layer = _ComposedLoaderLayer()
223
+ model = torch.nn.Sequential(layer)
224
+
225
+ def materialize_with_sentinel(meta_tensor):
226
+ tensor = torch.empty_strided(
227
+ size=tuple(meta_tensor.size()),
228
+ stride=tuple(meta_tensor.stride()),
229
+ dtype=meta_tensor.dtype,
230
+ requires_grad=False,
231
+ )
232
+ tensor.fill_(float("nan"))
233
+ tensor.__class__ = meta_tensor.__class__
234
+ tensor.__dict__ = meta_tensor.__dict__.copy()
235
+ return tensor
236
+
237
+ monkeypatch.setattr(
238
+ reload_meta, "materialize_meta_tensor", materialize_with_sentinel
239
+ )
240
+
241
+ loaded = {
242
+ "A": torch.full((4,), 0.5),
243
+ "dt_bias": torch.full((4,), 3.0),
244
+ "D": torch.full((4,), 7.0),
245
+ }
246
+
247
+ record_metadata_for_reloading(model)
248
+ initialize_layerwise_reload(model)
249
+ # Mimic real load_weights: resolve params once, then load in checkpoint
250
+ # order with D last (the param that was dropped).
251
+ params = dict(layer.named_parameters())
252
+ for name in ("A", "dt_bias", "D"):
253
+ param = params[name]
254
+ param.weight_loader(param, loaded[name])
255
+ finalize_layerwise_reload(model, model_config=None)
256
+
257
+ assert torch.equal(layer.A, -torch.exp(loaded["A"]))
258
+ assert torch.equal(layer.dt_bias, loaded["dt_bias"])
259
+ assert torch.equal(layer.D, loaded["D"])
260
+
261
+
262
+ def test_layerwise_reload_skips_non_persistent_parameter_alias_buffers(monkeypatch):
263
+ layer = _AliasedBufferLayer()
264
+ model = torch.nn.Sequential(layer)
265
+ loaded_weight = torch.full_like(layer.weight, 7.0)
266
+
267
+ def materialize_with_sentinel(meta_tensor):
268
+ tensor = torch.empty_strided(
269
+ size=tuple(meta_tensor.size()),
270
+ stride=tuple(meta_tensor.stride()),
271
+ dtype=meta_tensor.dtype,
272
+ requires_grad=False,
273
+ )
274
+ tensor.fill_(-123.0)
275
+ tensor.__class__ = meta_tensor.__class__
276
+ tensor.__dict__ = meta_tensor.__dict__.copy()
277
+ return tensor
278
+
279
+ monkeypatch.setattr(
280
+ reload_meta, "materialize_meta_tensor", materialize_with_sentinel
281
+ )
282
+
283
+ record_metadata_for_reloading(model)
284
+ initialize_layerwise_reload(model)
285
+ layer.weight.weight_loader(layer.weight, loaded_weight)
286
+ finalize_layerwise_reload(model, model_config=None)
287
+
288
+ assert torch.equal(layer.weight, loaded_weight)
289
+ assert layer.weight_view.untyped_storage().data_ptr() == (
290
+ layer.weight.untyped_storage().data_ptr()
291
+ )
292
+
293
+
294
+ def test_capture_layer_to_meta_skips_uninitialized_parameter_storage_ptrs():
295
+ layer = _AliasedBufferWithUninitializedChildLayer()
296
+
297
+ _, buffers = capture_layer_to_meta(layer)
298
+
299
+ assert "weight_view" not in buffers
300
+
301
+
302
+ def test_layerwise_reload_skips_child_parameter_alias_buffers(monkeypatch):
303
+ layer = _ParentAliasedChildBufferLayer()
304
+ model = torch.nn.Sequential(layer)
305
+ loaded_conv = torch.full_like(layer.conv1d.weight, 7.0)
306
+ loaded_scale = torch.full_like(layer.scale, 3.0)
307
+
308
+ def materialize_with_sentinel(meta_tensor):
309
+ tensor = torch.empty_strided(
310
+ size=tuple(meta_tensor.size()),
311
+ stride=tuple(meta_tensor.stride()),
312
+ dtype=meta_tensor.dtype,
313
+ requires_grad=False,
314
+ )
315
+ tensor.fill_(-123.0)
316
+ tensor.__class__ = meta_tensor.__class__
317
+ tensor.__dict__ = meta_tensor.__dict__.copy()
318
+ return tensor
319
+
320
+ monkeypatch.setattr(
321
+ reload_meta, "materialize_meta_tensor", materialize_with_sentinel
322
+ )
323
+
324
+ record_metadata_for_reloading(model)
325
+ initialize_layerwise_reload(model)
326
+ layer.conv1d.weight.weight_loader(layer.conv1d.weight, loaded_conv)
327
+ layer.scale.weight_loader(layer.scale, loaded_scale)
328
+ finalize_layerwise_reload(model, model_config=None)
329
+
330
+ assert torch.equal(layer.conv1d.weight, loaded_conv)
331
+ assert torch.equal(layer.conv_weights, loaded_conv.view(-1))
332
+ assert layer.conv_weights.untyped_storage().data_ptr() == (
333
+ layer.conv1d.weight.untyped_storage().data_ptr()
334
+ )
335
+
336
+
337
+ @pytest.mark.parametrize(
338
+ "tp_size", [pytest.param(1), pytest.param(2, marks=[pytest.mark.slow_test])]
339
+ )
340
+ @pytest.mark.parametrize(
341
+ "base_model,mul_model,add_model",
342
+ [
343
+ pytest.param(
344
+ "Qwen/Qwen3-0.6B",
345
+ "inference-optimization/Qwen3-0.6B-debug-multiply",
346
+ "inference-optimization/Qwen3-0.6B-debug-add",
347
+ marks=[pytest.mark.slow_test],
348
+ ),
349
+ pytest.param(
350
+ "inference-optimization/Qwen3-0.6B-FP8_BLOCK",
351
+ "inference-optimization/Qwen3-0.6B-debug-multiply-FP8_BLOCK",
352
+ "inference-optimization/Qwen3-0.6B-debug-add-FP8_BLOCK",
353
+ marks=[pytest.mark.slow_test],
354
+ ),
355
+ pytest.param(
356
+ "inference-optimization/Qwen3-0.6B-W4A16-G128",
357
+ "inference-optimization/Qwen3-0.6B-debug-multiply-W4A16-G128",
358
+ "inference-optimization/Qwen3-0.6B-debug-add-W4A16-G128",
359
+ marks=[pytest.mark.slow_test],
360
+ ),
361
+ pytest.param(
362
+ "inference-optimization/DeepSeek-V3-debug-empty",
363
+ "inference-optimization/DeepSeek-V3-debug-multiply",
364
+ "inference-optimization/DeepSeek-V3-debug-add",
365
+ marks=[pytest.mark.slow_test],
366
+ ),
367
+ pytest.param(
368
+ "inference-optimization/DeepSeek-V3-debug-empty-FP8_DYNAMIC",
369
+ "inference-optimization/DeepSeek-V3-debug-multiply-FP8_DYNAMIC",
370
+ "inference-optimization/DeepSeek-V3-debug-add-FP8_DYNAMIC",
371
+ ),
372
+ pytest.param(
373
+ "inference-optimization/DeepSeek-V3-debug-empty-NVFP4A16",
374
+ "inference-optimization/DeepSeek-V3-debug-multiply-NVFP4A16",
375
+ "inference-optimization/DeepSeek-V3-debug-add-NVFP4A16",
376
+ marks=[pytest.mark.slow_test],
377
+ ),
378
+ ],
379
+ )
380
+ def test_reload_weights(base_model, mul_model, add_model, tp_size, vllm_runner):
381
+ if current_platform.device_count() < tp_size:
382
+ pytest.skip(reason="Not enough CUDA devices")
383
+
384
+ if "FP8" in base_model and _fp8_reload_unsupported():
385
+ pytest.skip(reason="Requires FP8 support")
386
+
387
+ with vllm_runner(
388
+ model_name=base_model,
389
+ tensor_parallel_size=tp_size,
390
+ enable_expert_parallel=(tp_size > 1 and "DeepSeek" in base_model),
391
+ enable_prefix_caching=False,
392
+ max_model_len=16,
393
+ max_num_seqs=1,
394
+ ) as llm:
395
+ llm.collective_rpc("reload_weights", kwargs={"weights_path": mul_model})
396
+ mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
397
+ add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
398
+ assert mul_perp < add_perp
399
+
400
+ llm.collective_rpc("reload_weights", kwargs={"weights_path": add_model})
401
+ mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
402
+ add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
403
+ assert add_perp < mul_perp
404
+
405
+
406
+ def test_kv_scale_reload(vllm_runner):
407
+ """Test reloading a checkpoint that contains k_scale/v_scale weights."""
408
+ if _fp8_reload_unsupported():
409
+ pytest.skip(reason="Requires FP8 support")
410
+
411
+ model = "nm-testing/Llama-3.2-1B-Instruct-FP8-KV"
412
+
413
+ # Load dummy weights, then reload real checkpoint
414
+ with vllm_runner(
415
+ model_name=model,
416
+ load_format="dummy",
417
+ enable_prefix_caching=False,
418
+ max_model_len=16,
419
+ max_num_seqs=1,
420
+ ) as llm:
421
+ llm.collective_rpc(
422
+ "update_config",
423
+ kwargs={"overrides": {"load_config": {"load_format": "auto"}}},
424
+ )
425
+ llm.collective_rpc("reload_weights", kwargs={"weights_path": model})
426
+ reloaded_perp = llm.generate_prompt_perplexity(
427
+ ["The capital of France is the city of Paris"],
428
+ mask=["The capital of France is"],
429
+ )[0]
430
+
431
+ assert reloaded_perp < 10
432
+
433
+
434
+ @pytest.mark.parametrize(
435
+ "tp_size", [pytest.param(1), pytest.param(2, marks=[pytest.mark.slow_test])]
436
+ )
437
+ @pytest.mark.parametrize(
438
+ "base_model,mul_model,add_model,quantization",
439
+ [
440
+ pytest.param(
441
+ "Qwen/Qwen3-0.6B",
442
+ "inference-optimization/Qwen3-0.6B-debug-multiply",
443
+ "inference-optimization/Qwen3-0.6B-debug-add",
444
+ "fp8",
445
+ ),
446
+ pytest.param(
447
+ "inference-optimization/DeepSeek-V3-debug-empty",
448
+ "inference-optimization/DeepSeek-V3-debug-multiply",
449
+ "inference-optimization/DeepSeek-V3-debug-add",
450
+ "fp8",
451
+ marks=[pytest.mark.slow_test],
452
+ ),
453
+ pytest.param(
454
+ "Qwen/Qwen3-0.6B",
455
+ "inference-optimization/Qwen3-0.6B-debug-multiply",
456
+ "inference-optimization/Qwen3-0.6B-debug-add",
457
+ "mxfp8",
458
+ marks=[pytest.mark.slow_test],
459
+ ),
460
+ pytest.param(
461
+ "inference-optimization/DeepSeek-V3-debug-empty",
462
+ "inference-optimization/DeepSeek-V3-debug-multiply",
463
+ "inference-optimization/DeepSeek-V3-debug-add",
464
+ "mxfp8",
465
+ marks=[
466
+ pytest.mark.slow_test,
467
+ pytest.mark.xfail(reason="mxfp4 & mla is not supported yet"),
468
+ ],
469
+ ),
470
+ ],
471
+ )
472
+ def test_online_quantize_reload(
473
+ base_model, mul_model, add_model, quantization, tp_size, vllm_runner
474
+ ):
475
+ if current_platform.device_count() < tp_size:
476
+ pytest.skip(reason="Not enough GPU devices")
477
+
478
+ if quantization == "fp8" and _fp8_reload_unsupported():
479
+ pytest.skip(reason="Requires FP8 support")
480
+
481
+ with vllm_runner(
482
+ model_name=base_model,
483
+ quantization=quantization,
484
+ tensor_parallel_size=tp_size,
485
+ enable_expert_parallel=(tp_size > 1 and "DeepSeek" in base_model),
486
+ enable_prefix_caching=False,
487
+ max_model_len=16,
488
+ max_num_seqs=1,
489
+ ) as llm:
490
+ llm.collective_rpc("reload_weights", kwargs={"weights_path": mul_model})
491
+ mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
492
+ add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
493
+ assert mul_perp < add_perp
494
+
495
+ llm.collective_rpc("reload_weights", kwargs={"weights_path": add_model})
496
+ mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
497
+ add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
498
+ assert add_perp < mul_perp
asian_ai/performance/vLLM/tests/model_executor/model_loader/test_sharded_state_loader.py ADDED
@@ -0,0 +1,165 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import fnmatch
5
+ import multiprocessing as mp
6
+ import os
7
+ import shutil
8
+ from tempfile import TemporaryDirectory
9
+
10
+ import pytest
11
+ import torch
12
+ from huggingface_hub import snapshot_download
13
+
14
+ from vllm import LLM, SamplingParams
15
+ from vllm.model_executor.model_loader import ShardedStateLoader
16
+ from vllm.platforms import current_platform
17
+
18
+ prompts = [
19
+ "Hello, my name is",
20
+ "The president of the United States is",
21
+ "The capital of France is",
22
+ "The future of AI is",
23
+ ]
24
+
25
+ # Create a sampling params object.
26
+ sampling_params = SamplingParams(
27
+ temperature=0,
28
+ max_tokens=256,
29
+ ignore_eos=True,
30
+ )
31
+
32
+
33
+ def test_filter_subtensors():
34
+ state_dict = {
35
+ "a": torch.empty(2),
36
+ "b": torch.empty((2, 4)),
37
+ "c": torch.empty((2, 4, 8)),
38
+ }
39
+ state_dict.update(
40
+ {
41
+ "x": state_dict["b"],
42
+ "y": state_dict["c"][1, 2, :],
43
+ "z": state_dict["c"][1, :, 4],
44
+ }
45
+ )
46
+ filtered_state_dict = ShardedStateLoader._filter_subtensors(state_dict)
47
+ assert tuple(filtered_state_dict.keys()) == ("a", "b", "c")
48
+ for key, tensor in filtered_state_dict.items():
49
+ # NOTE: don't use `equal` here, as the tensor might contain NaNs
50
+ assert tensor is state_dict[key]
51
+
52
+
53
+ @pytest.fixture(scope="module")
54
+ def llama_3p2_1b_files():
55
+ input_dir = snapshot_download(
56
+ "meta-llama/Llama-3.2-1B-Instruct", ignore_patterns=["*.bin*", "original/*"]
57
+ )
58
+
59
+ yield input_dir
60
+
61
+
62
+ def _run_writer(input_dir, output_dir, weights_patterns, **kwargs):
63
+ llm_sharded_writer = LLM(model=input_dir, **kwargs)
64
+
65
+ # Dump worker states to output directory
66
+ llm_sharded_writer.llm_engine.engine_core.save_sharded_state(path=output_dir)
67
+
68
+ # Copy metadata files to output directory
69
+ for file in os.listdir(input_dir):
70
+ if os.path.isdir(os.path.join(input_dir, file)):
71
+ shutil.copytree(
72
+ os.path.join(input_dir, file), os.path.join(output_dir, file)
73
+ )
74
+ elif not any(fnmatch.fnmatch(file, ext) for ext in weights_patterns):
75
+ shutil.copy(os.path.join(input_dir, file), output_dir)
76
+
77
+
78
+ def _run_generate(input_dir, queue: mp.Queue, **kwargs):
79
+ llm = LLM(model=input_dir, **kwargs)
80
+ gen = llm.generate(prompts, sampling_params)
81
+ queue.put([g.outputs[0].__dict__ for g in gen])
82
+ queue.close()
83
+ queue.join_thread()
84
+
85
+
86
+ @pytest.mark.parametrize("enable_lora", [False, True])
87
+ @pytest.mark.parametrize("tp_size", [1, 2])
88
+ def test_sharded_state_loader(
89
+ enable_lora, tp_size, num_gpus_available, llama_3p2_1b_files
90
+ ):
91
+ if num_gpus_available < tp_size:
92
+ pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
93
+
94
+ weights_patterns = ("*.safetensors",)
95
+ gpu_memory_utilization = 0.8
96
+ input_dir = llama_3p2_1b_files
97
+ ctx = mp.get_context("spawn")
98
+
99
+ platform_args = {}
100
+ if current_platform.is_rocm() or current_platform.is_xpu():
101
+ platform_args["max_num_seqs"] = 1
102
+
103
+ # Run in separate processes for memory & CUDA isolation
104
+ with TemporaryDirectory() as output_dir:
105
+ p = ctx.Process(
106
+ target=_run_writer,
107
+ args=(input_dir, output_dir, weights_patterns),
108
+ kwargs=dict(
109
+ tensor_parallel_size=tp_size,
110
+ gpu_memory_utilization=gpu_memory_utilization,
111
+ enforce_eager=True,
112
+ **platform_args,
113
+ ),
114
+ )
115
+ p.start()
116
+ p.join()
117
+
118
+ queue = ctx.Queue()
119
+
120
+ p = ctx.Process(
121
+ target=_run_generate,
122
+ args=(input_dir, queue),
123
+ kwargs=dict(
124
+ enable_lora=enable_lora,
125
+ gpu_memory_utilization=gpu_memory_utilization,
126
+ tensor_parallel_size=tp_size,
127
+ **platform_args,
128
+ ),
129
+ )
130
+ p.start()
131
+ # Call queue.get() before p.join() to prevent deadlock:
132
+ # If p.join() is called before queue.get() and the queue is full,
133
+ # the child process may block while writing to the queue and never
134
+ # terminate, causing the parent to wait indefinitely on p.join().
135
+ # See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
136
+ out_before = queue.get()
137
+ p.join()
138
+ queue.close()
139
+ queue.join_thread()
140
+
141
+ queue = ctx.Queue()
142
+
143
+ p = ctx.Process(
144
+ target=_run_generate,
145
+ args=(output_dir, queue),
146
+ kwargs=dict(
147
+ enable_lora=enable_lora,
148
+ gpu_memory_utilization=gpu_memory_utilization,
149
+ tensor_parallel_size=tp_size,
150
+ load_format="sharded_state",
151
+ **platform_args,
152
+ ),
153
+ )
154
+ p.start()
155
+ # Call queue.get() before p.join() to prevent deadlock:
156
+ # If p.join() is called before queue.get() and the queue is full,
157
+ # the child process may block while writing to the queue and never
158
+ # terminate, causing the parent to wait indefinitely on p.join().
159
+ # See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
160
+ out_after = queue.get()
161
+ p.join()
162
+ queue.close()
163
+ queue.join_thread()
164
+
165
+ assert out_before == out_after
asian_ai/performance/vLLM/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py ADDED
@@ -0,0 +1,91 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ """Tests for CPU unquantized GEMM dispatch behavior."""
4
+
5
+ import pytest
6
+ import torch
7
+
8
+ from vllm.model_executor.layers import utils
9
+ from vllm.platforms import current_platform
10
+
11
+
12
+ @pytest.fixture(scope="module")
13
+ def _mock_zentorch_linear_unary():
14
+ """Register a mock zentorch_linear_unary op when zentorch is not installed.
15
+
16
+ Allows the dispatch tests to run in CI without a real zentorch build.
17
+ Skips registration when zentorch is already available.
18
+ """
19
+ if hasattr(torch.ops.zentorch, "zentorch_linear_unary"):
20
+ yield
21
+ return
22
+
23
+ lib_def = torch.library.Library("zentorch", "DEF")
24
+ lib_def.define(
25
+ "zentorch_linear_unary("
26
+ "Tensor input, "
27
+ "Tensor weight, "
28
+ "Tensor? bias, "
29
+ "bool is_weight_prepacked=False"
30
+ ") -> Tensor"
31
+ )
32
+
33
+ lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
34
+ lib_impl.impl(
35
+ "zentorch_linear_unary",
36
+ lambda input, weight, bias, is_weight_prepacked=False: (
37
+ torch.nn.functional.linear(input, weight, bias)
38
+ ),
39
+ )
40
+
41
+ yield
42
+
43
+ lib_impl._destroy()
44
+ lib_def._destroy()
45
+
46
+
47
+ @pytest.mark.usefixtures("_mock_zentorch_linear_unary")
48
+ def test_dispatch_cpu_unquantized_gemm_uses_zentorch_on_zen(monkeypatch):
49
+ monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
50
+
51
+ layer = torch.nn.Linear(16, 8, bias=True)
52
+ x = torch.randn(4, 16)
53
+ expected = torch.nn.functional.linear(x, layer.weight, layer.bias)
54
+
55
+ utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
56
+ output = layer.cpu_linear(x, layer.weight, layer.bias)
57
+
58
+ torch.testing.assert_close(output, expected)
59
+
60
+
61
+ @pytest.mark.usefixtures("_mock_zentorch_linear_unary")
62
+ def test_dispatch_cpu_unquantized_gemm_zen_remove_weight(monkeypatch):
63
+ monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
64
+
65
+ layer = torch.nn.Linear(16, 8, bias=True)
66
+ utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
67
+
68
+ assert layer.weight.numel() == 0
69
+
70
+
71
+ @pytest.mark.usefixtures("_mock_zentorch_linear_unary")
72
+ def test_dispatch_cpu_unquantized_gemm_logs_zentorch_dispatch(monkeypatch):
73
+ monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
74
+ expected_prepacked = bool(utils.envs.VLLM_ZENTORCH_WEIGHT_PREPACK) and hasattr(
75
+ torch.ops.zentorch, "zentorch_weight_prepack_for_linear"
76
+ )
77
+
78
+ log_calls = []
79
+ monkeypatch.setattr(
80
+ utils.logger, "debug_once", lambda *args: log_calls.append(args)
81
+ )
82
+
83
+ layer = torch.nn.Linear(16, 8, bias=True)
84
+ utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
85
+
86
+ assert log_calls == [
87
+ (
88
+ "CPU unquantized GEMM dispatch: using zentorch_linear_unary (prepacked=%s)",
89
+ expected_prepacked,
90
+ )
91
+ ]
asian_ai/performance/vLLM/tests/model_executor/test_eagle_quantization.py ADDED
@@ -0,0 +1,159 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ from unittest.mock import Mock, patch
5
+
6
+ import pytest
7
+ import torch
8
+
9
+ from vllm.config import LoadConfig, ModelConfig, SpeculativeConfig, VllmConfig
10
+ from vllm.model_executor.models.utils import get_draft_quant_config
11
+ from vllm.platforms import current_platform
12
+
13
+ DEVICE_TYPE = current_platform.device_type
14
+ DEVICES = (
15
+ [f"{DEVICE_TYPE}:{i}" for i in range(min(torch.accelerator.device_count(), 2))]
16
+ if not current_platform.is_cpu()
17
+ else ["cpu"]
18
+ )
19
+
20
+
21
+ def test_get_draft_quant_config_with_draft_model():
22
+ mock_draft_model_config = Mock(spec=ModelConfig)
23
+ mock_load_config = Mock(spec=LoadConfig)
24
+ mock_speculative_config = Mock(spec=SpeculativeConfig)
25
+ mock_speculative_config.draft_model_config = mock_draft_model_config
26
+
27
+ mock_vllm_config = Mock(spec=VllmConfig)
28
+ mock_vllm_config.speculative_config = mock_speculative_config
29
+ mock_vllm_config.load_config = mock_load_config
30
+
31
+ mock_quant_config = Mock()
32
+ with patch.object(
33
+ VllmConfig, "get_quantization_config", return_value=mock_quant_config
34
+ ):
35
+ result = get_draft_quant_config(mock_vllm_config)
36
+
37
+ # Verify the function calls get_quantization_config with draft model config
38
+ VllmConfig.get_quantization_config.assert_called_once_with(
39
+ mock_draft_model_config, mock_load_config
40
+ )
41
+ assert result == mock_quant_config
42
+
43
+
44
+ def test_get_draft_quant_config_without_draft_model():
45
+ mock_speculative_config = Mock(spec=SpeculativeConfig)
46
+ mock_speculative_config.draft_model_config = None
47
+
48
+ mock_vllm_config = Mock(spec=VllmConfig)
49
+ mock_vllm_config.speculative_config = mock_speculative_config
50
+ mock_vllm_config.load_config = Mock(spec=LoadConfig)
51
+
52
+ result = get_draft_quant_config(mock_vllm_config)
53
+
54
+ assert result is None
55
+
56
+
57
+ @torch.inference_mode()
58
+ @pytest.mark.parametrize("device", DEVICES)
59
+ def test_fc_layer_quant_config_usage(default_vllm_config, dist_init, device) -> None:
60
+ import torch
61
+
62
+ from vllm.model_executor.layers.linear import ReplicatedLinear
63
+
64
+ if current_platform.is_cuda_alike():
65
+ torch.accelerator.set_device_index(device)
66
+
67
+ torch.set_default_device(device)
68
+
69
+ input_size = 256
70
+ output_size = 128
71
+
72
+ fc_no_quant = ReplicatedLinear(
73
+ input_size=input_size,
74
+ output_size=output_size,
75
+ bias=False,
76
+ params_dtype=torch.float16,
77
+ quant_config=None,
78
+ prefix="fc",
79
+ )
80
+
81
+ assert fc_no_quant.quant_config is None
82
+ assert fc_no_quant.input_size == input_size
83
+ assert fc_no_quant.output_size == output_size
84
+
85
+ mock_quant_config = Mock()
86
+ fc_with_quant = ReplicatedLinear(
87
+ input_size=input_size,
88
+ output_size=output_size,
89
+ bias=False,
90
+ params_dtype=torch.float16,
91
+ quant_config=mock_quant_config,
92
+ prefix="fc",
93
+ )
94
+
95
+ assert fc_with_quant.quant_config == mock_quant_config
96
+
97
+ # Check forward pass
98
+ x = torch.randn(2, input_size, dtype=torch.float16)
99
+ output, _ = fc_no_quant(x)
100
+ assert output.shape == (2, output_size)
101
+
102
+
103
+ def test_maybe_remap_kv_scale_name():
104
+ from vllm.model_executor.model_loader.weight_utils import maybe_remap_kv_scale_name
105
+
106
+ params_dict = {
107
+ "layers.0.self_attn.kv_scale": Mock(),
108
+ "layers.1.self_attn.kv_scale": Mock(),
109
+ }
110
+
111
+ name = "layers.0.self_attn.some_scale"
112
+ remapped = maybe_remap_kv_scale_name(name, params_dict)
113
+
114
+ assert remapped in params_dict or remapped == name or remapped is None
115
+
116
+
117
+ def test_eagle3_lm_head_receives_quant_config():
118
+ """Eagle3LlamaForCausalLM must pass quant_config to ParallelLMHead.
119
+
120
+ Without quant_config, quantized lm_head weights (e.g. INT8 per-channel)
121
+ in Eagle3 drafter checkpoints fail to load because ParallelLMHead doesn't
122
+ expect weight_packed tensors.
123
+ """
124
+ from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
125
+
126
+ mock_quant_config = Mock()
127
+
128
+ mock_hf_config = Mock()
129
+ mock_hf_config.draft_vocab_size = 1000
130
+ mock_hf_config.hidden_size = 256
131
+ mock_hf_config.vocab_size = 32000
132
+ mock_hf_config.logit_scale = 1.0
133
+
134
+ mock_vllm_config = Mock()
135
+ mock_vllm_config.speculative_config.draft_model_config.hf_config = mock_hf_config
136
+ mock_vllm_config.model_config.get_num_layers.return_value = 32
137
+ mock_vllm_config.speculative_config.parallel_drafting = False
138
+
139
+ with (
140
+ patch("vllm.model_executor.models.llama_eagle3.LlamaModel") as MockModel,
141
+ patch("vllm.model_executor.models.llama_eagle3.ParallelLMHead") as MockLMHead,
142
+ patch("vllm.model_executor.models.llama_eagle3.LogitsProcessor"),
143
+ patch(
144
+ "vllm.model_executor.models.llama_eagle3.get_draft_quant_config",
145
+ return_value=mock_quant_config,
146
+ ),
147
+ ):
148
+ MockModel.return_value.use_aux_hidden_state = True
149
+
150
+ Eagle3LlamaForCausalLM(vllm_config=mock_vllm_config)
151
+
152
+ MockLMHead.assert_called_once()
153
+ call_kwargs = MockLMHead.call_args.kwargs
154
+ assert "quant_config" in call_kwargs, (
155
+ "ParallelLMHead must receive quant_config for quantized lm_head weights"
156
+ )
157
+ assert call_kwargs["quant_config"] is mock_quant_config, (
158
+ "ParallelLMHead must receive the draft model's quant_config"
159
+ )
asian_ai/performance/vLLM/tests/model_executor/test_enabled_custom_ops.py ADDED
@@ -0,0 +1,151 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+
4
+ import pytest
5
+ import torch
6
+
7
+ from vllm._aiter_ops import rocm_aiter_ops
8
+ from vllm.config import (
9
+ CompilationConfig,
10
+ VllmConfig,
11
+ get_cached_compilation_config,
12
+ set_current_vllm_config,
13
+ )
14
+ from vllm.model_executor.custom_op import CustomOp, op_registry
15
+ from vllm.model_executor.layers.activation import (
16
+ GeluAndMul,
17
+ ReLUSquaredActivation,
18
+ SiluAndMul,
19
+ )
20
+ from vllm.model_executor.layers.fused_moe.router.fused_topk_router import (
21
+ dispatch_topk_sigmoid_func,
22
+ dispatch_topk_softmax_func,
23
+ vllm_topk_sigmoid,
24
+ vllm_topk_softmax,
25
+ )
26
+ from vllm.model_executor.layers.layernorm import RMSNorm
27
+ from vllm.platforms import current_platform
28
+
29
+ RMS_NORM_SUPPORTED_DTYPES = [torch.float16, torch.bfloat16]
30
+
31
+
32
+ # Registered subclass for test
33
+ @CustomOp.register("relu3")
34
+ class Relu3(ReLUSquaredActivation):
35
+ pass
36
+
37
+
38
+ @pytest.mark.parametrize(
39
+ "env, compilation_mode, backend, ops_enabled, default_on",
40
+ [
41
+ # Default values based on compile level
42
+ # - All by default (no Inductor compilation)
43
+ (None, 0, "eager", [True] * 4, True),
44
+ (None, 1, "eager", [True] * 4, True),
45
+ (None, 2, "eager", [True] * 4, True),
46
+ (None, 3, "eager", [True] * 4, True),
47
+ # - None by default (with Inductor)
48
+ (None, 0, "inductor", [True] * 4, True),
49
+ # - None by default (with Inductor)
50
+ (None, 1, "inductor", [False] * 4, False),
51
+ (None, 2, "inductor", [False] * 4, False),
52
+ (None, 3, "inductor", [False] * 4, False),
53
+ # Explicitly enabling/disabling
54
+ #
55
+ # Default: all
56
+ #
57
+ # All but SiluAndMul
58
+ ("+rms_norm,-silu_and_mul", 0, "inductor", [1, 0, 1, 1], True),
59
+ # Only ReLU3
60
+ ("none,-rms_norm,+relu3", 1, "eager", [0, 0, 0, 1], False),
61
+ # All but SiluAndMul
62
+ ("all,-silu_and_mul", 2, "inductor", [1, 0, 1, 1], True),
63
+ # All but ReLU3 (even if ReLU2 is on)
64
+ ("-relu3,+relu2", 3, "eager", [1, 1, 1, 0], True),
65
+ # RMSNorm and SiluAndMul
66
+ ("none,-relu3,+rms_norm,+silu_and_mul", 3, "eager", [1, 1, 0, 0], False),
67
+ # All but RMSNorm
68
+ ("-rms_norm", 3, "eager", [0, 1, 1, 1], True),
69
+ #
70
+ # Default: none
71
+ #
72
+ # Only ReLU3
73
+ ("none,+relu3", 3, "inductor", [0, 0, 0, 1], False),
74
+ # All but RMSNorm
75
+ ("all,-rms_norm", 3, "inductor", [0, 1, 1, 1], True),
76
+ ],
77
+ )
78
+ def test_enabled_ops(
79
+ env: str | None,
80
+ compilation_mode: int,
81
+ backend: str,
82
+ ops_enabled: list[int],
83
+ default_on: bool,
84
+ ):
85
+ custom_ops = env.split(",") if env else []
86
+ vllm_config = VllmConfig(
87
+ compilation_config=CompilationConfig(
88
+ backend=backend, mode=compilation_mode, custom_ops=custom_ops
89
+ )
90
+ )
91
+ get_cached_compilation_config.cache_clear()
92
+ with set_current_vllm_config(vllm_config):
93
+ assert CustomOp.default_on() == default_on
94
+
95
+ ops_enabled = [bool(x) for x in ops_enabled]
96
+
97
+ assert RMSNorm(1024).enabled() == ops_enabled[0]
98
+ assert op_registry["rms_norm"].enabled() == ops_enabled[0]
99
+
100
+ assert SiluAndMul().enabled() == ops_enabled[1]
101
+ assert op_registry["silu_and_mul"].enabled() == ops_enabled[1]
102
+
103
+ assert GeluAndMul().enabled() == ops_enabled[2]
104
+ assert op_registry["gelu_and_mul"].enabled() == ops_enabled[2]
105
+
106
+ # If registered, subclasses should follow their own name
107
+ assert Relu3().enabled() == ops_enabled[3]
108
+ assert op_registry["relu3"].enabled() == ops_enabled[3]
109
+
110
+ # Unregistered subclass
111
+ class SiluAndMul2(SiluAndMul):
112
+ pass
113
+
114
+ # Subclasses should not require registration
115
+ assert SiluAndMul2().enabled() == SiluAndMul().enabled()
116
+
117
+
118
+ @pytest.mark.parametrize(
119
+ "env", ["all,none", "all,+rms_norm,all", "+rms_norm,-rms_norm"]
120
+ )
121
+ def test_enabled_ops_invalid(env: str):
122
+ with pytest.raises(Exception): # noqa
123
+ vllm_config = VllmConfig(
124
+ compilation_config=CompilationConfig(custom_ops=env.split(","))
125
+ )
126
+ with set_current_vllm_config(vllm_config):
127
+ RMSNorm(1024).enabled()
128
+
129
+
130
+ @pytest.mark.parametrize(
131
+ "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
132
+ )
133
+ def test_topk_softmax_dispatch(use_rocm_aiter: bool):
134
+ topk_func = dispatch_topk_softmax_func(use_rocm_aiter)
135
+
136
+ if current_platform.is_rocm() and use_rocm_aiter:
137
+ assert topk_func == rocm_aiter_ops.topk_softmax
138
+ else:
139
+ assert topk_func == vllm_topk_softmax
140
+
141
+
142
+ @pytest.mark.parametrize(
143
+ "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
144
+ )
145
+ def test_topk_sigmoid_dispatch(use_rocm_aiter: bool):
146
+ topk_func = dispatch_topk_sigmoid_func(use_rocm_aiter)
147
+
148
+ if current_platform.is_rocm() and use_rocm_aiter:
149
+ assert topk_func == rocm_aiter_ops.topk_sigmoid
150
+ else:
151
+ assert topk_func == vllm_topk_sigmoid
asian_ai/performance/vLLM/tests/model_executor/test_ernie45_vl_mrope.py ADDED
@@ -0,0 +1,143 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # SPDX-License-Identifier: Apache-2.0
2
+ # SPDX-FileCopyrightText: Copyright contributors to the vLLM project
3
+ from dataclasses import dataclass
4
+
5
+ import pytest
6
+ import torch
7
+
8
+ from vllm.model_executor.models.ernie45_vl import (
9
+ Ernie4_5_VLMoeForConditionalGeneration,
10
+ )
11
+ from vllm.multimodal.inputs import (
12
+ MultiModalFeatureSpec,
13
+ MultiModalFieldElem,
14
+ MultiModalKwargsItem,
15
+ PlaceholderRange,
16
+ )
17
+
18
+ pytestmark = pytest.mark.skip_global_cleanup
19
+
20
+
21
+ @pytest.fixture(autouse=True, scope="module")
22
+ def _force_cpu_default_device():
23
+ original = torch.get_default_device()
24
+ torch.set_default_device("cpu")
25
+ yield
26
+ torch.set_default_device(original)
27
+
28
+
29
+ @dataclass
30
+ class DummyConfig:
31
+ spatial_conv_size: int = 2
32
+ temporal_conv_size: int = 2
33
+
34
+
35
+ def make_model(config: DummyConfig) -> Ernie4_5_VLMoeForConditionalGeneration:
36
+ model = object.__new__(Ernie4_5_VLMoeForConditionalGeneration)
37
+ model.config = config
38
+ return model
39
+
40
+
41
+ def make_mm_feature(
42
+ *,
43
+ modality: str,
44
+ offset: int,
45
+ length: int,
46
+ grid_thw: tuple[int, int, int],
47
+ ) -> MultiModalFeatureSpec:
48
+ field_name = "image_grid_thw" if modality == "image" else "video_grid_thw"
49
+ return MultiModalFeatureSpec(
50
+ data=MultiModalKwargsItem(
51
+ {
52
+ field_name: MultiModalFieldElem(
53
+ data=torch.tensor(grid_thw),
54
+ field=None, # HACK.
55
+ ),
56
+ }
57
+ ),
58
+ modality=modality,
59
+ identifier="DUMMY",
60
+ mm_position=PlaceholderRange(offset=offset, length=length),
61
+ )
62
+
63
+
64
+ def test_get_mrope_input_positions_text_only():
65
+ model = make_model(DummyConfig())
66
+
67
+ positions, delta = model.get_mrope_input_positions(
68
+ input_tokens=[11, 12, 13, 14, 15],
69
+ mm_features=[],
70
+ )
71
+
72
+ expected = torch.tensor(
73
+ [
74
+ [0, 1, 2, 3, 4],
75
+ [0, 1, 2, 3, 4],
76
+ [0, 1, 2, 3, 4],
77
+ ]
78
+ )
79
+
80
+ assert torch.equal(positions, expected)
81
+ assert delta == 0
82
+
83
+
84
+ def test_get_mrope_input_positions_single_image():
85
+ model = make_model(DummyConfig())
86
+ mm_features = [
87
+ make_mm_feature(
88
+ modality="image",
89
+ offset=1,
90
+ length=4,
91
+ grid_thw=(1, 4, 4),
92
+ )
93
+ ]
94
+
95
+ positions, delta = model.get_mrope_input_positions(
96
+ input_tokens=[10, 20, 21, 22, 23, 30, 31],
97
+ mm_features=mm_features,
98
+ )
99
+
100
+ expected = torch.tensor(
101
+ [
102
+ [0, 1, 1, 1, 1, 3, 4],
103
+ [0, 1, 1, 2, 2, 3, 4],
104
+ [0, 1, 2, 1, 2, 3, 4],
105
+ ]
106
+ )
107
+
108
+ assert torch.equal(positions, expected)
109
+ assert delta == -2
110
+
111
+
112
+ def test_get_mrope_input_positions_interleaved_image_and_video():
113
+ model = make_model(DummyConfig())
114
+ mm_features = [
115
+ make_mm_feature(
116
+ modality="image",
117
+ offset=1,
118
+ length=4,
119
+ grid_thw=(1, 4, 4),
120
+ ),
121
+ make_mm_feature(
122
+ modality="video",
123
+ offset=7,
124
+ length=2,
125
+ grid_thw=(2, 4, 2),
126
+ ),
127
+ ]
128
+
129
+ positions, delta = model.get_mrope_input_positions(
130
+ input_tokens=[10, 20, 21, 22, 23, 30, 31, 40, 41, 50, 51],
131
+ mm_features=mm_features,
132
+ )
133
+
134
+ expected = torch.tensor(
135
+ [
136
+ [0, 1, 1, 1, 1, 3, 4, 5, 5, 7, 8],
137
+ [0, 1, 1, 2, 2, 3, 4, 5, 6, 7, 8],
138
+ [0, 1, 2, 1, 2, 3, 4, 5, 5, 7, 8],
139
+ ]
140
+ )
141
+
142
+ assert torch.equal(positions, expected)
143
+ assert delta == -2