diff --git a/.gitattributes b/.gitattributes
index d7c5dae6046fa92da28de8b85f8d201ca67252cf..d1121e25f7a77088b39e97fb63a3beb2d5618ffb 100644
--- a/.gitattributes
+++ b/.gitattributes
@@ -880,3 +880,6 @@ asian_ai/performance/vLLM/docs/assets/features/disagg_prefill/overview.jpg filte
asian_ai/performance/vLLM/docs/assets/training/layerwise.png filter=lfs diff=lfs merge=lfs -text
asian_ai/performance/vLLM/docs/assets/training/layerwise_bad_loading.png filter=lfs diff=lfs merge=lfs -text
asian_ai/performance/vLLM/docs/assets/training/layerwise_good_loading.png filter=lfs diff=lfs merge=lfs -text
+asian_ai/performance/vLLM/tests/multimodal/assets/rgba.png filter=lfs diff=lfs merge=lfs -text
+asian_ai/performance/vLLM/tests/v1/ec_connector/integration/hato.jpg filter=lfs diff=lfs merge=lfs -text
+asian_ai/performance/vLLM/tools/profiler/nsys_profile_tools/images/csv1.png filter=lfs diff=lfs merge=lfs -text
diff --git a/asian_ai/performance/vLLM/tests/lora/test_moe_lora_ep_load.py b/asian_ai/performance/vLLM/tests/lora/test_moe_lora_ep_load.py
new file mode 100644
index 0000000000000000000000000000000000000000..e24ea486a79c4912d302fa30d3c115f8873a5cf1
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_moe_lora_ep_load.py
@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+End-to-end correctness test for 2D MoE LoRA expert-parallel
+load-time slicing
+"""
+
+import pytest
+import torch
+
+from vllm.lora.lora_model import LoRAModel, MoEEPLoadSpec
+from vllm.lora.peft_helper import PEFTHelper
+
+NUM_LAYERS = 48
+GLOBAL_NUM_EXPERTS = 128
+LOCAL_NUM_EXPERTS = 64 # ep_size = 2
+EXPERT_PROJECTIONS = ("down_proj", "gate_proj", "up_proj")
+NON_EXPERT_MODULES = ("q_proj", "k_proj", "v_proj", "o_proj", "gate")
+
+
+def _expected_lora_modules() -> set[str]:
+ """Replicate the set ``WorkerLoRAManager._load_adapter`` would build
+ from this model's ``packed_modules_mapping``."""
+ expected: set[str] = set(NON_EXPERT_MODULES)
+ for expert in range(GLOBAL_NUM_EXPERTS):
+ for proj in EXPERT_PROJECTIONS:
+ expected.add(f"experts.{expert}.{proj}")
+ return expected
+
+
+def _load(lora_dir, peft_helper, *, moe_ep_spec, lora_id):
+ return LoRAModel.from_local_checkpoint(
+ lora_dir,
+ _expected_lora_modules(),
+ peft_helper=peft_helper,
+ lora_model_id=lora_id,
+ device="cpu",
+ moe_ep_spec=moe_ep_spec,
+ )
+
+
+@pytest.mark.parametrize("ep_rank", [0, 1])
+def test_moe_lora_ep2_real_qwen3moe(qwen3moe_lora_files, ep_rank):
+ """ep_size=2 against the real Qwen3-MoE adapter: each rank's loaded
+ LoRA has the right size, the right expert membership, and the
+ right tensor values."""
+ peft_helper = PEFTHelper.from_local_dir(
+ qwen3moe_lora_files, max_position_embeddings=4096
+ )
+
+ # Baseline: no spec → loads every expert × projection × layer plus
+ # all non-expert LoRA modules.
+ ground_truth = _load(qwen3moe_lora_files, peft_helper, moe_ep_spec=None, lora_id=1)
+ expected_baseline = (
+ GLOBAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
+ + len(NON_EXPERT_MODULES) * NUM_LAYERS
+ )
+ assert len(ground_truth.loras) == expected_baseline
+
+ # Sliced load: only this rank's experts; non-expert LoRA is untouched.
+ spec = MoEEPLoadSpec(
+ ep_rank=ep_rank,
+ local_num_experts=LOCAL_NUM_EXPERTS,
+ global_num_experts=GLOBAL_NUM_EXPERTS,
+ )
+ sliced = _load(
+ qwen3moe_lora_files,
+ peft_helper,
+ moe_ep_spec=spec,
+ lora_id=100 + ep_rank,
+ )
+
+ expected_sliced = (
+ LOCAL_NUM_EXPERTS * len(EXPERT_PROJECTIONS) * NUM_LAYERS
+ + len(NON_EXPERT_MODULES) * NUM_LAYERS
+ )
+ assert len(sliced.loras) == expected_sliced
+
+ expert_start = ep_rank * LOCAL_NUM_EXPERTS
+ expert_end = expert_start + LOCAL_NUM_EXPERTS
+
+ for name, lora in sliced.loras.items():
+ gt = ground_truth.loras[name]
+ torch.testing.assert_close(lora.lora_a, gt.lora_a)
+ torch.testing.assert_close(lora.lora_b, gt.lora_b)
+ if ".experts." in name:
+ expert_idx = int(name.split(".experts.")[-1].split(".")[0])
+ assert expert_start <= expert_idx < expert_end, (
+ f"non-local expert {expert_idx} leaked: {name}"
+ )
diff --git a/asian_ai/performance/vLLM/tests/lora/test_olmoe_tp.py b/asian_ai/performance/vLLM/tests/lora/test_olmoe_tp.py
new file mode 100644
index 0000000000000000000000000000000000000000..ceb871e08b48ae66db6d0a294cff2125de829a45
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_olmoe_tp.py
@@ -0,0 +1,226 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+
+import shutil
+from collections.abc import Sequence
+
+import pytest
+import torch
+from safetensors.torch import load_file, save_file
+
+import vllm
+from vllm.lora.request import LoRARequest
+from vllm.platforms import current_platform
+
+from ..utils import multi_gpu_test
+
+MODEL_PATH = "allenai/OLMoE-1B-7B-0125-Instruct"
+
+PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me. Do not return any additional explanation. Below is an instruction that describes a task, Write a response that appropriately completes the request.
+"
+##Instruction:
+candidate_poll contains tables such as candidate, people. Table candidate has columns such as Candidate_ID, People_ID, Poll_Source, Date, Support_rate, Consider_rate, Oppose_rate, Unsure_rate. Candidate_ID is the primary key.
+Table people has columns such as People_ID, Sex, Name, Date_of_Birth, Height, Weight. People_ID is the primary key.
+The People_ID of candidate is the foreign key of People_ID of people.
+
+
+###Input:
+{context}
+
+###Response:""" # noqa: E501
+
+EXPECTED_LORA_OUTPUT = [
+ "SELECT count(*) FROM candidate",
+ "SELECT count(*) FROM candidate",
+ "SELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1", # noqa: E501
+ "SELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1", # noqa: E501
+]
+
+EXPECTED_BASE_MODEL_OUTPUT = [
+ "SELECT COUNT(Candidate_ID) FROM candidate",
+ "SELECT COUNT(Candidate_ID) FROM candidate",
+ "SELECT Candidate_ID, COUNT(*) as Total_Candidates\nFROM candidate\nINNER JOIN people ON candidate.People_ID = people.People_ID", # noqa: E501
+ # There are multiple acceptable responses
+ (
+ "SELECT Candidate_ID, Poll_Source FROM candidate WHERE People_ID IN (SELECT People_ID FROM people) ORDER BY COUNT(*) DESC LIMIT 1", # noqa: E501
+ "SELECT Candidate_ID, Poll_Source FROM candidate WHERE COUNT(People_ID) = (SELECT COUNT(People_ID) FROM people) ORDER BY Candidate_ID DESC LIMIT 1", # noqa: E501
+ ),
+]
+
+
+def _output_matches(generated: str, accepted: str | Sequence[str]) -> bool:
+ if isinstance(accepted, str):
+ accepted = (accepted,)
+ return any(generated.startswith(s) for s in accepted)
+
+
+def generate_and_test(
+ llm: vllm.LLM,
+ lora_path: str,
+ lora_id: list[int | None] | int | None,
+ compare_lower: bool = False,
+) -> None:
+ prompts = [
+ PROMPT_TEMPLATE.format(context="How many candidates are there?"),
+ PROMPT_TEMPLATE.format(context="Count the number of candidates."),
+ PROMPT_TEMPLATE.format(
+ context="Which poll resource provided the most number of candidate information?" # noqa: E501
+ ),
+ PROMPT_TEMPLATE.format(
+ context="Return the poll resource associated with the most candidates."
+ ),
+ ]
+
+ lora_request = None
+ if isinstance(lora_id, int):
+ lora_request = LoRARequest(str(lora_id), lora_id, lora_path)
+ elif isinstance(lora_id, list):
+ lora_request = [
+ LoRARequest(str(i), i, lora_path) if i is not None else None
+ for i in lora_id
+ ]
+
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=64)
+ outputs = llm.generate(prompts, sampling_params, lora_request=lora_request)
+ # Print the outputs.
+ generated_texts: list[str] = []
+ for output in outputs:
+ prompt = output.prompt
+ generated_text = output.outputs[0].text.strip()
+ generated_texts.append(generated_text)
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
+ req_lora_id = lora_id[i] if isinstance(lora_id, list) else lora_id
+ generated_text = generated_texts[i]
+ expected_output = (
+ EXPECTED_LORA_OUTPUT[i]
+ if req_lora_id is not None
+ else EXPECTED_BASE_MODEL_OUTPUT[i]
+ )
+
+ if compare_lower:
+ generated_text = generated_text.lower()
+ if isinstance(expected_output, str):
+ expected_output = (expected_output.lower(),)
+ else:
+ expected_output = tuple(s.lower() for s in expected_output)
+ assert _output_matches(generated_text, expected_output), (
+ f"Output {i}: {generated_text!r} does not match any of {expected_output!r}"
+ )
+
+
+@pytest.mark.skipif(
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
+)
+def test_olmoe_lora(olmoe_lora_files, maybe_enable_lora_dual_stream):
+ # We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
+ # Otherwise, the lora-test will fail due to CUDA OOM.
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enable_lora=True,
+ max_loras=4,
+ enforce_eager=True,
+ trust_remote_code=True,
+ enable_chunked_prefill=True,
+ )
+
+ generate_and_test(llm, olmoe_lora_files, lora_id=1)
+ generate_and_test(llm, olmoe_lora_files, lora_id=2)
+
+
+def test_olmoe_lora_mixed(olmoe_lora_files):
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enable_lora=True,
+ max_loras=4,
+ enforce_eager=True,
+ trust_remote_code=True,
+ enable_chunked_prefill=True,
+ )
+
+ generate_and_test(llm, olmoe_lora_files, lora_id=[1, None, 3, None])
+
+
+def test_olmoe_lora_mixed_random(
+ olmoe_lora_files, tmp_path, maybe_enable_lora_dual_stream
+):
+ # Create a dummy LoRA with random weights based on the real one
+ random_lora_path = tmp_path / "random_lora"
+ shutil.copytree(olmoe_lora_files, random_lora_path)
+
+ weights_path = random_lora_path / "adapter_model.safetensors"
+ weights = load_file(str(weights_path))
+ random_weights = {k: torch.randn_like(v) for k, v in weights.items()}
+ save_file(random_weights, str(weights_path))
+
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enable_lora=True,
+ max_loras=4,
+ enforce_eager=True,
+ trust_remote_code=True,
+ enable_chunked_prefill=True,
+ )
+
+ prompts = [
+ PROMPT_TEMPLATE.format(context="How many candidates are there?"),
+ PROMPT_TEMPLATE.format(context="Count the number of candidates."),
+ ]
+
+ lora_requests = [
+ LoRARequest("real", 1, olmoe_lora_files),
+ LoRARequest("random", 2, str(random_lora_path)),
+ ]
+
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=64)
+ outputs = llm.generate(prompts, sampling_params, lora_request=lora_requests)
+ assert outputs[0].outputs[0].text.strip().startswith(EXPECTED_LORA_OUTPUT[0])
+
+
+@pytest.mark.skipif(
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
+)
+@pytest.mark.parametrize("fully_sharded_loras", [False, True])
+@multi_gpu_test(num_gpus=2)
+def test_olmoe_lora_tp2(olmoe_lora_files, fully_sharded_loras):
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enable_lora=True,
+ max_loras=4,
+ enforce_eager=True,
+ trust_remote_code=True,
+ enable_chunked_prefill=True,
+ tensor_parallel_size=2,
+ fully_sharded_loras=fully_sharded_loras,
+ )
+
+ generate_and_test(llm, olmoe_lora_files, lora_id=1)
+ generate_and_test(llm, olmoe_lora_files, lora_id=2)
+
+
+@pytest.mark.parametrize("fully_sharded_loras", [False, True])
+@multi_gpu_test(num_gpus=4)
+def test_olmoe_lora_tp4(olmoe_lora_files, fully_sharded_loras):
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enable_lora=True,
+ max_loras=4,
+ enforce_eager=True,
+ trust_remote_code=True,
+ enable_chunked_prefill=True,
+ tensor_parallel_size=4,
+ fully_sharded_loras=fully_sharded_loras,
+ )
+ generate_and_test(
+ llm, olmoe_lora_files, lora_id=1, compare_lower=fully_sharded_loras
+ )
+ generate_and_test(
+ llm, olmoe_lora_files, lora_id=2, compare_lower=fully_sharded_loras
+ )
diff --git a/asian_ai/performance/vLLM/tests/lora/test_peft_helper.py b/asian_ai/performance/vLLM/tests/lora/test_peft_helper.py
new file mode 100644
index 0000000000000000000000000000000000000000..baa971e7c7a4cdec808b605e459330b489076b81
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_peft_helper.py
@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import json
+import math
+import shutil
+
+import pytest
+
+from vllm.config.lora import LoRAConfig
+from vllm.lora.peft_helper import PEFTHelper
+
+ERROR_CASES = [
+ (
+ "test_rank",
+ {"r": 1024},
+ "is greater than max_lora_rank",
+ ),
+ ("test_dora", {"use_dora": True}, "does not yet support DoRA"),
+ (
+ "test_modules_to_save",
+ {"modules_to_save": ["lm_head"]},
+ "only supports modules_to_save being None",
+ ),
+]
+
+
+def test_peft_helper_pass(llama32_lora_files, tmp_path):
+ peft_helper = PEFTHelper.from_local_dir(
+ llama32_lora_files, max_position_embeddings=4096
+ )
+ lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
+ peft_helper.validate_legal(lora_config)
+ assert peft_helper.r == 8
+ assert peft_helper.lora_alpha == 32
+ target_modules = sorted(peft_helper.target_modules)
+
+ assert target_modules == [
+ "down_proj",
+ "embed_tokens",
+ "gate_proj",
+ "k_proj",
+ "lm_head",
+ "o_proj",
+ "q_proj",
+ "up_proj",
+ "v_proj",
+ ]
+ assert peft_helper.vllm_max_position_embeddings == 4096
+
+ # test RSLoRA
+ rslora_config = dict(use_rslora=True)
+ test_dir = tmp_path / "test_rslora"
+ shutil.copytree(llama32_lora_files, test_dir)
+
+ # Load and modify configuration
+ config_path = test_dir / "adapter_config.json"
+ with open(config_path) as f:
+ adapter_config = json.load(f)
+ # Apply configuration changes
+ adapter_config.update(rslora_config)
+
+ # Save modified configuration
+ with open(config_path, "w") as f:
+ json.dump(adapter_config, f)
+
+ peft_helper = PEFTHelper.from_local_dir(test_dir, max_position_embeddings=4096)
+ peft_helper.validate_legal(lora_config)
+ scaling = peft_helper.lora_alpha / math.sqrt(peft_helper.r)
+ assert abs(peft_helper.vllm_lora_scaling_factor - scaling) < 1e-3
+
+
+@pytest.mark.parametrize("test_name,config_change,expected_error", ERROR_CASES)
+def test_peft_helper_error(
+ llama32_lora_files,
+ tmp_path,
+ test_name: str,
+ config_change: dict,
+ expected_error: str,
+):
+ test_dir = tmp_path / test_name
+ shutil.copytree(llama32_lora_files, test_dir)
+
+ # Load and modify configuration
+ config_path = test_dir / "adapter_config.json"
+ with open(config_path) as f:
+ adapter_config = json.load(f)
+ # Apply configuration changes
+ adapter_config.update(config_change)
+
+ # Save modified configuration
+ with open(config_path, "w") as f:
+ json.dump(adapter_config, f)
+ lora_config = LoRAConfig(max_lora_rank=16, max_cpu_loras=3, max_loras=2)
+ # Test loading the adapter
+ with pytest.raises(ValueError, match=expected_error):
+ PEFTHelper.from_local_dir(
+ test_dir, max_position_embeddings=4096
+ ).validate_legal(lora_config)
diff --git a/asian_ai/performance/vLLM/tests/lora/test_punica_ops.py b/asian_ai/performance/vLLM/tests/lora/test_punica_ops.py
new file mode 100644
index 0000000000000000000000000000000000000000..b9b6489f33bca2d1a79f5cb7b2996ef9a0ffee1a
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_punica_ops.py
@@ -0,0 +1,608 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from threading import Lock
+
+import pytest
+import torch
+
+import vllm.lora.ops.torch_ops as torch_ops
+import vllm.lora.ops.triton_ops as triton_ops
+from vllm.lora.ops.triton_ops import LoRAKernelMeta
+from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
+from vllm.platforms import current_platform
+from vllm.utils.torch_utils import set_random_seed
+
+from .utils import PunicaTensors, assert_close, generate_data_for_nslices
+
+DEVICE_TYPE = current_platform.device_type
+
+
+@pytest.fixture(autouse=True)
+def reset_device(reset_default_device):
+ pass
+
+
+# Utility shrink and expand operations used as reference implementations.
+def sgmv_shrink_for_nslices(
+ nslices: int,
+ inputs_tensor: torch.Tensor,
+ lora_weights_lst: list[torch.Tensor],
+ out_tensor: torch.Tensor,
+ b_seq_start_loc: torch.Tensor,
+ seq_len_tensor: torch.Tensor,
+ prompt_lora_mapping: torch.Tensor,
+ batches: int,
+ max_seq_length: int,
+ num_tokens: int,
+ scaling: float,
+):
+ """
+ Wrapper around torch_ops.sgmv_shrink that handles any nslices.
+ """
+ for index in range(nslices):
+ torch_ops.sgmv_shrink(
+ inputs_tensor,
+ lora_weights_lst[index],
+ out_tensor[index],
+ b_seq_start_loc,
+ seq_len_tensor,
+ prompt_lora_mapping,
+ batches,
+ max_seq_length,
+ num_tokens,
+ scaling,
+ )
+
+
+def sgmv_expand_for_nslices(
+ nslices: int,
+ hidden_size: int,
+ inputs_tensor: torch.Tensor,
+ lora_weights_lst: list[torch.Tensor],
+ out_tensor: torch.Tensor,
+ b_seq_start_loc: torch.Tensor,
+ seq_len_tensor: torch.Tensor,
+ prompt_lora_mapping: torch.Tensor,
+ batches: int,
+ max_seq_length: int,
+ num_tokens: int,
+ add_inputs: bool,
+) -> None:
+ """
+ Wrapper around torch_ops.sgmv_expand that handles any nslices.
+ """
+ if nslices == 1:
+ # Verify the torch's sgmv_expand op
+ torch_ops.sgmv_expand(
+ inputs_tensor[0],
+ lora_weights_lst[0],
+ out_tensor,
+ b_seq_start_loc,
+ seq_len_tensor,
+ prompt_lora_mapping,
+ batches,
+ max_seq_length,
+ num_tokens,
+ add_inputs=add_inputs,
+ )
+ else:
+ slice_offset = 0
+ for index in range(nslices):
+ lora_weights = lora_weights_lst[index]
+ torch_ops.sgmv_expand_slice(
+ inputs_tensor[index],
+ lora_weights,
+ out_tensor,
+ b_seq_start_loc,
+ seq_len_tensor,
+ prompt_lora_mapping,
+ batches,
+ max_seq_length,
+ num_tokens,
+ slice_offset,
+ hidden_size,
+ add_inputs=add_inputs,
+ )
+ slice_offset += hidden_size
+
+
+_dict_lock = Lock()
+
+
+def check_lora_shrink_kernel(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ nslices: int,
+ dtype: torch.dtype,
+ device: str,
+ seq_length: int,
+ scaling: float,
+):
+ """
+ Compare outputs of torch_ops.sgmv_shrink and triton_ops.lora_shrink
+ kernels.
+ """
+ data: PunicaTensors = generate_data_for_nslices(
+ batches,
+ hidden_size,
+ num_loras,
+ rank,
+ seq_length,
+ nslices,
+ dtype,
+ "shrink",
+ device,
+ )
+ max_seq_length, token_nums = data.meta()
+
+ # Setup metadata information for SGMV and reference kernels
+ sgmv_meta_args = (
+ data.b_seq_start_loc,
+ data.seq_len_tensor,
+ data.prompt_lora_mapping,
+ batches,
+ max_seq_length,
+ token_nums,
+ )
+
+ # Setup metadata information for the LoRA kernel.
+ lora_meta = LoRAKernelMeta.make(
+ max_loras=num_loras,
+ max_num_tokens=token_nums,
+ device=DEVICE_TYPE,
+ )
+ lora_meta.prepare_tensors(data.token_lora_mapping)
+
+ ref_out_tensor = data.ref_out_tensor
+ out_tensor = data.our_out_tensor.clone()
+
+ # Preventing cache error pointer.
+ with _dict_lock:
+ # lora_shrink kernel
+ _LORA_A_PTR_DICT.clear()
+ triton_ops.lora_shrink(
+ data.inputs_tensor,
+ data.lora_weights,
+ out_tensor,
+ *lora_meta.meta_args(token_nums=token_nums, specialize_active_lora=False),
+ scaling,
+ )
+
+ # Reference
+ sgmv_shrink_for_nslices(
+ nslices,
+ data.inputs_tensor,
+ data.lora_weights,
+ ref_out_tensor,
+ *sgmv_meta_args,
+ scaling,
+ )
+
+ assert_close(out_tensor, ref_out_tensor)
+
+
+def check_lora_expand_kernel(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ nslices: int,
+ dtype: torch.dtype,
+ device: str,
+ seq_length: int,
+ add_inputs: bool,
+):
+ """
+ Compare outputs of torch_ops.sgmv_expand and triton_ops.lora_expand
+ kernels.
+ """
+ data: PunicaTensors = generate_data_for_nslices(
+ batches,
+ hidden_size,
+ num_loras,
+ rank,
+ seq_length,
+ nslices,
+ dtype,
+ "expand",
+ device,
+ )
+
+ max_seq_length, token_nums = data.meta()
+
+ # Setup metadata information for SGMV and reference kernels
+ sgmv_meta_args = (
+ data.b_seq_start_loc,
+ data.seq_len_tensor,
+ data.prompt_lora_mapping,
+ batches,
+ max_seq_length,
+ token_nums,
+ )
+
+ # Setup metadata information for the LoRA kernel.
+ lora_meta = LoRAKernelMeta.make(
+ max_loras=num_loras,
+ max_num_tokens=token_nums,
+ device=DEVICE_TYPE,
+ )
+ lora_meta.prepare_tensors(data.token_lora_mapping)
+
+ # Setup output tensors
+ ref_out_tensor = data.ref_out_tensor
+ out_tensor = data.our_out_tensor.clone()
+
+ with _dict_lock:
+ # lora_expand kernel
+ _LORA_B_PTR_DICT.clear()
+ triton_ops.lora_expand(
+ data.inputs_tensor,
+ data.lora_weights,
+ out_tensor,
+ *lora_meta.meta_args(token_nums=token_nums, specialize_active_lora=False),
+ offset_start=0,
+ add_inputs=add_inputs,
+ )
+
+ # Reference
+ sgmv_expand_for_nslices(
+ nslices,
+ hidden_size,
+ data.inputs_tensor,
+ data.lora_weights,
+ ref_out_tensor,
+ *sgmv_meta_args,
+ add_inputs=add_inputs,
+ )
+
+ assert_close(out_tensor, ref_out_tensor)
+
+
+# Tests
+# We test the punica kernels along 2 verticals mainly.
+# 1. Variations in hidden_dim size
+# 2. Variations in all other parameters like (batch_size, max_rank, num_loras
+# etc.)
+
+# We have collected the hidden_sizes included in the LoRA models
+# currently supported by vLLM. It tests whether the corresponding Triton
+# kernel can run normally when tensor parallelism is set to
+# [1, 2, 4, 8, 16, 32, 64].
+HIDDEN_SIZES = [
+ 128,
+ 256,
+ 512,
+ 896,
+ 1024,
+ 1152,
+ 1216,
+ 1280,
+ 1536,
+ 1664,
+ 2048,
+ 2240,
+ 2304,
+ 2368,
+ 2432,
+ 2560,
+ 2752,
+ 3072,
+ 3328,
+ 3456,
+ 3584,
+ 3712,
+ 4096,
+ 4480,
+ 4608,
+ 4736,
+ 4864,
+ 5120,
+ 5504,
+ 5632,
+ 5888,
+ 6144,
+ 6400,
+ 6848,
+ 6912,
+ 7168,
+ 7424,
+ 8192,
+ 8960,
+ 9216,
+ 9472,
+ 10240,
+ 11008,
+ 11264,
+ 13824,
+ 14336,
+ 14784,
+ 14848,
+ 15360,
+ 18944,
+ 22016,
+ 22528,
+ 24576,
+ 27392,
+ 27648,
+ 29568,
+ 29696,
+ 32000,
+ 32256,
+ 32512,
+ 32768,
+ 33024,
+ 36864,
+ 43264,
+ 49152,
+ 49408,
+ 60544,
+ 60672,
+ 64000,
+ 64256,
+ 102400,
+ 102656,
+ 128000,
+ 128256,
+]
+# The size of TP
+divisibility = [1, 2, 8, 16, 64]
+
+all_hidden_size = []
+for div in divisibility:
+ for hidden_size in HIDDEN_SIZES:
+ all_hidden_size.append(hidden_size // div)
+
+HIDDEN_SIZES = list(set(all_hidden_size))
+
+# Test params that focuses on hidden_size variation.
+hs_test_params = {
+ "hidden_sizes": HIDDEN_SIZES,
+ "batches": [4],
+ "num_loras": [4],
+ "max_ranks": [32],
+}
+
+# General tests params that tests for variations in all dimensions
+# except hidden_size.
+test_params = {
+ "hidden_sizes": [2049],
+ "batches": [1, 4, 16, 32],
+ "num_loras": [1, 8, 32, 128],
+ "max_ranks": [1, 4, 8, 16, 32, 64, 128, 256],
+}
+
+DTYPES = [torch.float16, torch.bfloat16]
+DEVICES = [f"{DEVICE_TYPE}:{0}"]
+SEED = [0]
+
+
+@pytest.mark.parametrize("batches", test_params["batches"])
+@pytest.mark.parametrize("num_loras", test_params["num_loras"])
+@pytest.mark.parametrize("rank", test_params["max_ranks"])
+@pytest.mark.parametrize("hidden_size", test_params["hidden_sizes"])
+@pytest.mark.parametrize("nslices", [1, 2, 3])
+@pytest.mark.parametrize("dtype", DTYPES)
+@pytest.mark.parametrize("device", DEVICES)
+@pytest.mark.parametrize("seed", SEED)
+@pytest.mark.parametrize("op_type", ["shrink", "expand"])
+def test_kernels(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ nslices: int,
+ dtype: torch.dtype,
+ device: str,
+ seed: int,
+ op_type: str,
+):
+ """
+ Tests LoRA kernels.
+ """
+ torch.set_default_device(device)
+ torch.accelerator.set_device_index(device)
+ set_random_seed(seed)
+
+ if op_type == "shrink":
+ check_lora_shrink_kernel(
+ batches=batches,
+ num_loras=num_loras,
+ rank=rank,
+ hidden_size=hidden_size,
+ nslices=nslices,
+ dtype=dtype,
+ device=device,
+ seq_length=128,
+ scaling=0.5,
+ )
+ else:
+ check_lora_expand_kernel(
+ batches=batches,
+ num_loras=num_loras,
+ rank=rank,
+ hidden_size=hidden_size,
+ nslices=nslices,
+ dtype=dtype,
+ device=device,
+ seq_length=128,
+ add_inputs=True,
+ )
+
+
+@pytest.mark.parametrize("batches", hs_test_params["batches"])
+@pytest.mark.parametrize("num_loras", hs_test_params["num_loras"])
+@pytest.mark.parametrize("rank", hs_test_params["max_ranks"])
+@pytest.mark.parametrize("hidden_size", hs_test_params["hidden_sizes"])
+@pytest.mark.parametrize("nslices", [1, 2, 3])
+@pytest.mark.parametrize("dtype", DTYPES)
+@pytest.mark.parametrize("device", DEVICES)
+@pytest.mark.parametrize("seed", SEED)
+@pytest.mark.parametrize("op_type", ["shrink", "expand"])
+def test_kernels_hidden_size(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ nslices: int,
+ dtype: torch.dtype,
+ device: str,
+ seed: int,
+ op_type: str,
+):
+ """
+ Tests SGMV and LoRA kernels.
+ """
+ torch.set_default_device(device)
+ torch.accelerator.set_device_index(device)
+ set_random_seed(seed)
+
+ if op_type == "shrink":
+ check_lora_shrink_kernel(
+ batches=batches,
+ num_loras=num_loras,
+ rank=rank,
+ hidden_size=hidden_size,
+ nslices=nslices,
+ dtype=dtype,
+ device=device,
+ seq_length=128,
+ scaling=0.5,
+ )
+ else:
+ check_lora_expand_kernel(
+ batches=batches,
+ num_loras=num_loras,
+ rank=rank,
+ hidden_size=hidden_size,
+ nslices=nslices,
+ dtype=dtype,
+ device=device,
+ seq_length=128,
+ add_inputs=True,
+ )
+
+
+@pytest.mark.parametrize("device", DEVICES)
+def test_add_lora_fused_moe_early_exit(device):
+ """
+ Ensures add_lora_fused_moe does not invoke the LoRA kernel or
+ modify the output tensor when no_lora_flag_cpu is True
+ """
+ from types import SimpleNamespace
+
+ from vllm.lora.punica_wrapper.punica_gpu import PunicaWrapperGPU
+
+ torch.set_default_device(device)
+ torch.accelerator.set_device_index(device)
+
+ max_loras, num_tokens = 4, 16
+ num_experts, top_k, max_lora_rank = 8, 2, 16
+ K, N = 256, 128
+
+ # build PunicaWrapperGPU with minimal lora_config mock
+ lora_config = SimpleNamespace(
+ max_loras=max_loras,
+ specialize_active_lora=False,
+ )
+ wrapper = PunicaWrapperGPU(
+ max_num_batched_tokens=num_tokens,
+ max_batches=num_tokens,
+ device=device,
+ lora_config=lora_config,
+ )
+
+ # simulate a prior LoRA batch so the internal mapping is
+ # populated with stale LoRA IDs
+ lora_mapping = torch.zeros(
+ num_tokens,
+ dtype=torch.int32,
+ device=device,
+ )
+ lora_mapping[:8] = 1
+ lora_mapping[8:] = 2
+ wrapper.token_mapping_meta.prepare_tensors(lora_mapping)
+
+ # simulate a base-model batch (all -1)
+ base_mapping = torch.full(
+ (num_tokens,),
+ -1,
+ dtype=torch.int32,
+ device=device,
+ )
+ wrapper.token_mapping_meta.prepare_tensors(base_mapping)
+
+ assert wrapper.token_mapping_meta.no_lora_flag_cpu[0].item() is True
+
+ # dummy tensors for add_lora_fused_moe
+ y = torch.rand(num_tokens, top_k, N, dtype=torch.bfloat16, device=device)
+ y_snapshot = y.clone()
+ x = torch.rand(num_tokens, K, dtype=torch.bfloat16, device=device)
+
+ lora_a_stacked = (
+ torch.rand(
+ max_loras,
+ num_experts,
+ max_lora_rank,
+ K,
+ dtype=torch.bfloat16,
+ device=device,
+ ),
+ )
+ lora_b_stacked = (
+ torch.rand(
+ max_loras,
+ num_experts,
+ N,
+ max_lora_rank,
+ dtype=torch.bfloat16,
+ device=device,
+ ),
+ )
+ topk_weights = torch.ones(
+ num_tokens,
+ top_k,
+ dtype=torch.float32,
+ device=device,
+ )
+ adapter_enabled = torch.ones(
+ max_loras + 1,
+ dtype=torch.int32,
+ device=device,
+ )
+ shrink_config = expand_config = {
+ "BLOCK_SIZE_M": 16,
+ "BLOCK_SIZE_N": 32,
+ "BLOCK_SIZE_K": 64,
+ "GROUP_SIZE_M": 1,
+ "NUM_WARPS": 4,
+ "NUM_STAGES": 3,
+ "SPLIT_K": 1,
+ }
+
+ # call add_lora_fused_moe - the early exit should prevent any
+ # modification to the output
+ wrapper.add_lora_fused_moe(
+ y=y,
+ x=x,
+ lora_a_stacked=lora_a_stacked,
+ lora_b_stacked=lora_b_stacked,
+ topk_weights=topk_weights,
+ sorted_token_ids=None,
+ expert_ids=torch.zeros(
+ num_tokens * top_k,
+ dtype=torch.int32,
+ device=device,
+ ),
+ num_tokens_post_padded=None,
+ max_lora_rank=max_lora_rank,
+ top_k_num=top_k,
+ shrink_config=shrink_config,
+ expand_config=expand_config,
+ adapter_enabled=adapter_enabled,
+ )
+
+ assert torch.equal(y, y_snapshot), (
+ "add_lora_fused_moe modified output tensor despite no_lora_flag_cpu=True"
+ )
diff --git a/asian_ai/performance/vLLM/tests/lora/test_punica_ops_fp8.py b/asian_ai/performance/vLLM/tests/lora/test_punica_ops_fp8.py
new file mode 100644
index 0000000000000000000000000000000000000000..9e7106e907b20d11f5e3972b4f66c6ca9ba92214
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_punica_ops_fp8.py
@@ -0,0 +1,1001 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""FP8 accuracy tests for LoRA shrink and expand kernels.
+
+Tests the FP8 kernels by:
+1. Quantizing bf16 inputs/weights to FP8
+2. Dequantizing them back to bf16
+3. Running the bf16 reference (sgmv_shrink/sgmv_expand) with dequantized values
+4. Comparing FP8 kernel output against this dequantized reference
+
+This isolates kernel correctness from quantization precision loss,
+allowing much tighter tolerances than comparing against the original bf16.
+"""
+
+import math
+from threading import Lock
+
+import pytest
+import torch
+
+import vllm.lora.ops.torch_ops as torch_ops
+import vllm.lora.ops.triton_ops as triton_ops
+from vllm.lora.ops.triton_ops import LoRAKernelMeta
+from vllm.lora.ops.triton_ops.lora_expand_fp8_op import (
+ _EXPAND_LORA_SCALE_PTR_DICT,
+)
+from vllm.lora.ops.triton_ops.lora_shrink_fp8_op import (
+ _SHRINK_LORA_SCALE_PTR_DICT,
+)
+from vllm.lora.ops.triton_ops.utils import _LORA_A_PTR_DICT, _LORA_B_PTR_DICT
+from vllm.platforms import current_platform
+from vllm.utils.torch_utils import set_random_seed
+
+DEVICE_TYPE = current_platform.device_type
+DEVICES = [f"{DEVICE_TYPE}:{0}"]
+SEED = [0]
+
+_dict_lock = Lock()
+
+
+@pytest.fixture(autouse=True)
+def reset_device(reset_default_device):
+ pass
+
+
+# ============================================================================
+# Reference implementations (bf16 baseline)
+# ============================================================================
+
+
+def sgmv_shrink_for_nslices(
+ nslices,
+ inputs_tensor,
+ lora_weights_lst,
+ out_tensor,
+ b_seq_start_loc,
+ seq_len_tensor,
+ prompt_lora_mapping,
+ batches,
+ max_seq_length,
+ num_tokens,
+ scaling,
+):
+ """Wrapper around torch_ops.sgmv_shrink that handles any nslices."""
+ for index in range(nslices):
+ torch_ops.sgmv_shrink(
+ inputs_tensor,
+ lora_weights_lst[index],
+ out_tensor[index],
+ b_seq_start_loc,
+ seq_len_tensor,
+ prompt_lora_mapping,
+ batches,
+ max_seq_length,
+ num_tokens,
+ scaling,
+ )
+
+
+def sgmv_expand_for_nslices(
+ nslices,
+ hidden_size,
+ inputs_tensor,
+ lora_weights_lst,
+ out_tensor,
+ b_seq_start_loc,
+ seq_len_tensor,
+ prompt_lora_mapping,
+ batches,
+ max_seq_length,
+ num_tokens,
+ add_inputs,
+):
+ """Wrapper around torch_ops.sgmv_expand that handles any nslices."""
+ if nslices == 1:
+ torch_ops.sgmv_expand(
+ inputs_tensor[0],
+ lora_weights_lst[0],
+ out_tensor,
+ b_seq_start_loc,
+ seq_len_tensor,
+ prompt_lora_mapping,
+ batches,
+ max_seq_length,
+ num_tokens,
+ add_inputs=add_inputs,
+ )
+ else:
+ slice_offset = 0
+ for index in range(nslices):
+ torch_ops.sgmv_expand_slice(
+ inputs_tensor[index],
+ lora_weights_lst[index],
+ out_tensor,
+ b_seq_start_loc,
+ seq_len_tensor,
+ prompt_lora_mapping,
+ batches,
+ max_seq_length,
+ num_tokens,
+ slice_offset,
+ hidden_size,
+ add_inputs=add_inputs,
+ )
+ slice_offset += hidden_size
+
+
+# ============================================================================
+# FP8 Quantization Helpers
+# ============================================================================
+
+FP8_DTYPE = torch.float8_e4m3fn
+FP8_MAX = torch.finfo(FP8_DTYPE).max
+FP8_MIN = torch.finfo(FP8_DTYPE).min
+
+
+def quantize_to_fp8_per_tensor(
+ tensor: torch.Tensor,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Quantize a tensor to FP8 with per-tensor scaling."""
+ amax = tensor.abs().float().max().clamp(min=1e-12)
+ scale = (amax / FP8_MAX).to(torch.float32)
+ fp8_tensor = (tensor.float() / scale).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
+ return fp8_tensor, scale.reshape(1)
+
+
+def quantize_to_fp8_per_channel(
+ tensor: torch.Tensor,
+ channel_dim: int = 0,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Quantize a tensor to FP8 with per-channel scaling.
+
+ For shrink lora_a weights of shape (num_loras, rank, hidden_size):
+ channel_dim=1 gives per-rank scaling -> scale shape (num_loras, rank)
+ For expand lora_b weights of shape (num_loras, hidden_size, rank):
+ channel_dim=1 gives per-hidden scaling -> scale shape (num_loras, hidden_size)
+ """
+ # Compute amax along all dims except the leading dims up to channel_dim+1
+ reduce_dims = list(range(channel_dim + 1, tensor.ndim))
+ if reduce_dims:
+ amax = tensor.abs().float().amax(dim=reduce_dims).clamp(min=1e-12)
+ else:
+ amax = tensor.abs().float().clamp(min=1e-12)
+ scale = (amax / FP8_MAX).to(torch.float32)
+
+ # Expand scale for broadcasting
+ for _ in reduce_dims:
+ scale = scale.unsqueeze(-1)
+ fp8_tensor = (tensor.float() / scale).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
+ scale = scale.squeeze()
+ if scale.ndim == 0:
+ scale = scale.unsqueeze(0)
+ return fp8_tensor, scale
+
+
+def quantize_to_fp8_per_token(
+ tensor: torch.Tensor,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Quantize a 2D tensor to FP8 with per-token (per-row) scaling.
+
+ Input shape: (num_tokens, hidden_size)
+ Returns: (fp8_tensor, scale) where scale shape is (num_tokens, 1)
+ """
+ assert tensor.ndim == 2
+ amax = tensor.abs().float().amax(dim=1, keepdim=True).clamp(min=1e-12)
+ scale = (amax / FP8_MAX).to(torch.float32)
+ fp8_tensor = (tensor.float() / scale).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
+ return fp8_tensor, scale
+
+
+def quantize_to_fp8_blockwise(
+ tensor: torch.Tensor,
+ group_n: int,
+ group_k: int,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Quantize a 2D or 3D tensor to FP8 with block-wise scaling.
+
+ For a 2D tensor (num_tokens, hidden_size):
+ Blocks of size (1, group_k) ->
+ scale shape (num_tokens, ceil(hidden_size/group_k))
+
+ For a 3D tensor (num_loras, N, K):
+ Blocks of size (group_n, group_k) ->
+ scale shape (num_loras, ceil(N/group_n), ceil(K/group_k))
+ """
+ if tensor.ndim == 2:
+ M, K = tensor.shape
+ n_blocks_k = math.ceil(K / group_k)
+ scale = torch.zeros(M, n_blocks_k, dtype=torch.float32, device=tensor.device)
+ fp8_tensor = torch.zeros_like(tensor, dtype=FP8_DTYPE)
+ for m in range(M):
+ for bk in range(n_blocks_k):
+ k_start = bk * group_k
+ k_end = min(k_start + group_k, K)
+ block = tensor[m, k_start:k_end].float()
+ amax = block.abs().max().clamp(min=1e-12)
+ s = (amax / FP8_MAX).to(torch.float32)
+ scale[m, bk] = s
+ fp8_tensor[m, k_start:k_end] = (
+ (block / s).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
+ )
+ return fp8_tensor, scale
+ elif tensor.ndim == 3:
+ L, N, K = tensor.shape
+ n_blocks_n = math.ceil(N / group_n)
+ n_blocks_k = math.ceil(K / group_k)
+ scale = torch.zeros(
+ L, n_blocks_n, n_blocks_k, dtype=torch.float32, device=tensor.device
+ )
+ fp8_tensor = torch.zeros_like(tensor, dtype=FP8_DTYPE)
+ for li in range(L):
+ for bn in range(n_blocks_n):
+ for bk in range(n_blocks_k):
+ n_start = bn * group_n
+ n_end = min(n_start + group_n, N)
+ k_start = bk * group_k
+ k_end = min(k_start + group_k, K)
+ block = tensor[li, n_start:n_end, k_start:k_end].float()
+ amax = block.abs().max().clamp(min=1e-12)
+ s = (amax / FP8_MAX).to(torch.float32)
+ scale[li, bn, bk] = s
+ fp8_tensor[li, n_start:n_end, k_start:k_end] = (
+ (block / s).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
+ )
+ return fp8_tensor, scale
+ else:
+ raise ValueError(f"Unsupported tensor ndim: {tensor.ndim}")
+
+
+# ============================================================================
+# FP8 Dequantization Helpers
+# ============================================================================
+
+
+def dequantize_fp8_per_tensor(
+ fp8_tensor: torch.Tensor,
+ scale: torch.Tensor,
+ output_dtype: torch.dtype = torch.bfloat16,
+) -> torch.Tensor:
+ """Dequantize FP8 tensor with per-tensor scale back to output_dtype."""
+ return (fp8_tensor.float() * scale.float()).to(output_dtype)
+
+
+def dequantize_fp8_per_channel(
+ fp8_tensor: torch.Tensor,
+ scale: torch.Tensor,
+ channel_dim: int,
+ output_dtype: torch.dtype = torch.bfloat16,
+) -> torch.Tensor:
+ """Dequantize FP8 tensor with per-channel scale back to output_dtype.
+
+ For 3D tensor (num_loras, N, K) with channel_dim=1:
+ scale shape is (num_loras, N), broadcast over K.
+ """
+ expand_scale = scale.float()
+ # Add trailing dims for broadcasting
+ for _ in range(channel_dim + 1, fp8_tensor.ndim):
+ expand_scale = expand_scale.unsqueeze(-1)
+ return (fp8_tensor.float() * expand_scale).to(output_dtype)
+
+
+def dequantize_fp8_per_token(
+ fp8_tensor: torch.Tensor,
+ scale: torch.Tensor,
+ output_dtype: torch.dtype = torch.bfloat16,
+) -> torch.Tensor:
+ """Dequantize FP8 2D tensor with per-token scale back to output_dtype.
+
+ fp8_tensor: (num_tokens, hidden_size), scale: (num_tokens, 1)
+ """
+ return (fp8_tensor.float() * scale.float()).to(output_dtype)
+
+
+def dequantize_fp8_blockwise(
+ fp8_tensor: torch.Tensor,
+ scale: torch.Tensor,
+ group_n: int,
+ group_k: int,
+ output_dtype: torch.dtype = torch.bfloat16,
+) -> torch.Tensor:
+ """Dequantize FP8 tensor with block-wise scale back to output_dtype."""
+ if fp8_tensor.ndim == 2:
+ M, K = fp8_tensor.shape
+ out = torch.zeros(M, K, dtype=output_dtype, device=fp8_tensor.device)
+ n_blocks_k = math.ceil(K / group_k)
+ for m in range(M):
+ for bk in range(n_blocks_k):
+ k_start = bk * group_k
+ k_end = min(k_start + group_k, K)
+ out[m, k_start:k_end] = (
+ fp8_tensor[m, k_start:k_end].float() * scale[m, bk].float()
+ ).to(output_dtype)
+ return out
+ elif fp8_tensor.ndim == 3:
+ L, N, K = fp8_tensor.shape
+ out = torch.zeros(L, N, K, dtype=output_dtype, device=fp8_tensor.device)
+ n_blocks_n = math.ceil(N / group_n)
+ n_blocks_k = math.ceil(K / group_k)
+ for l_idx in range(L):
+ for bn in range(n_blocks_n):
+ for bk in range(n_blocks_k):
+ n_start = bn * group_n
+ n_end = min(n_start + group_n, N)
+ k_start = bk * group_k
+ k_end = min(k_start + group_k, K)
+ out[l_idx, n_start:n_end, k_start:k_end] = (
+ fp8_tensor[l_idx, n_start:n_end, k_start:k_end].float()
+ * scale[l_idx, bn, bk].float()
+ ).to(output_dtype)
+ return out
+ else:
+ raise ValueError(f"Unsupported tensor ndim: {fp8_tensor.ndim}")
+
+
+# ============================================================================
+# FP8 Data Generation
+# ============================================================================
+
+
+def generate_fp8_shrink_data(
+ batches: int,
+ hidden_size: int,
+ num_loras: int,
+ rank: int,
+ seq_length: int,
+ nslices: int,
+ dtype: torch.dtype,
+ device: str,
+ quant_mode: str, # "per_tensor", "per_channel", "blockwise"
+ group_k: int = 128,
+ group_n: int = 128,
+):
+ """Generate test data for FP8 shrink kernel.
+
+ Shrink: output = input @ lora_a^T * scaling
+ input: (num_tokens, hidden_size) -> quantized to FP8
+ lora_a: (num_loras, rank, hidden_size) -> quantized to FP8
+
+ Returns bf16 reference tensors, FP8 quantized tensors with scales,
+ and dequantized bf16 tensors for accurate reference computation.
+ """
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1, (batches,)).to(device)
+ b_seq_start_loc = torch.cumsum(
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
+ dim=0,
+ ).to(device)
+ total_tokens = seq_len_tensor.sum().item()
+
+ # Generate bf16 reference data
+ inputs_bf16 = torch.randn(total_tokens, hidden_size, dtype=dtype, device=device)
+
+ lora_a_weights_bf16 = []
+ for _ in range(nslices):
+ lora_a_weights_bf16.append(
+ torch.randn(num_loras, rank, hidden_size, dtype=dtype, device=device)
+ )
+
+ # Quantize inputs to FP8 and dequantize back for reference
+ if quant_mode == "blockwise":
+ inputs_fp8, a_scale = quantize_to_fp8_blockwise(
+ inputs_bf16, group_n=1, group_k=group_k
+ )
+ inputs_dequant = dequantize_fp8_blockwise(
+ inputs_fp8,
+ a_scale,
+ group_n=1,
+ group_k=group_k,
+ output_dtype=dtype,
+ )
+ elif quant_mode == "per_tensor":
+ # Per-tensor: kernel loads a single scalar from a_scale_ptr
+ inputs_fp8, a_scale = quantize_to_fp8_per_tensor(inputs_bf16)
+ inputs_dequant = dequantize_fp8_per_tensor(
+ inputs_fp8,
+ a_scale,
+ output_dtype=dtype,
+ )
+ else:
+ # per_channel: kernel loads per-token a_scale via ram indexing
+ inputs_fp8, a_scale = quantize_to_fp8_per_token(inputs_bf16)
+ inputs_dequant = dequantize_fp8_per_token(
+ inputs_fp8,
+ a_scale,
+ output_dtype=dtype,
+ )
+
+ # Quantize lora_a weights to FP8 and dequantize back for reference
+ b_scales = []
+ lora_a_weights_fp8 = []
+ lora_a_weights_dequant = []
+ for w in lora_a_weights_bf16:
+ if quant_mode == "per_tensor":
+ w_fp8, w_scale = quantize_to_fp8_per_tensor(w)
+ w_dequant = dequantize_fp8_per_tensor(w_fp8, w_scale, output_dtype=dtype)
+ # Scale shape: (1,) -> need (num_loras,) for the kernel
+ w_scale = w_scale.expand(num_loras).contiguous()
+ lora_a_weights_fp8.append(w_fp8)
+ b_scales.append(w_scale)
+ lora_a_weights_dequant.append(w_dequant)
+ elif quant_mode == "per_channel":
+ # Per-channel along rank dim: scale shape (num_loras, rank)
+ w_fp8, w_scale = quantize_to_fp8_per_channel(w, channel_dim=1)
+ w_dequant = dequantize_fp8_per_channel(
+ w_fp8,
+ w_scale,
+ channel_dim=1,
+ output_dtype=dtype,
+ )
+ lora_a_weights_fp8.append(w_fp8)
+ b_scales.append(w_scale)
+ lora_a_weights_dequant.append(w_dequant)
+ elif quant_mode == "blockwise":
+ w_fp8, w_scale = quantize_to_fp8_blockwise(
+ w, group_n=group_n, group_k=group_k
+ )
+ w_dequant = dequantize_fp8_blockwise(
+ w_fp8,
+ w_scale,
+ group_n=group_n,
+ group_k=group_k,
+ output_dtype=dtype,
+ )
+ lora_a_weights_fp8.append(w_fp8)
+ b_scales.append(w_scale)
+ lora_a_weights_dequant.append(w_dequant)
+
+ # Output tensor (float32 for shrink)
+ out_tensor = torch.zeros(
+ nslices, total_tokens, rank, dtype=torch.float32, device=device
+ )
+ ref_out_tensor = out_tensor.clone()
+
+ # Token-to-lora mapping
+ lora_indices_tensor = torch.randint(0, max(num_loras - 1, 1), (batches,)).to(device)
+ token_lora_mapping = torch.zeros(total_tokens, dtype=torch.long, device=device)
+ current_offset = 0
+ for b_id in range(batches):
+ lora_index = lora_indices_tensor[b_id]
+ sl = seq_len_tensor[b_id].item()
+ token_lora_mapping[current_offset : current_offset + sl] = lora_index
+ current_offset += sl
+
+ return {
+ "inputs_bf16": inputs_bf16,
+ "inputs_fp8": inputs_fp8,
+ "inputs_dequant": inputs_dequant,
+ "lora_a_bf16": lora_a_weights_bf16,
+ "lora_a_fp8": lora_a_weights_fp8,
+ "lora_a_dequant": lora_a_weights_dequant,
+ "a_scale": a_scale,
+ "b_scales": b_scales,
+ "out_tensor": out_tensor,
+ "ref_out_tensor": ref_out_tensor,
+ "token_lora_mapping": token_lora_mapping,
+ "seq_len_tensor": seq_len_tensor,
+ "b_seq_start_loc": b_seq_start_loc,
+ "lora_indices_tensor": lora_indices_tensor,
+ "total_tokens": total_tokens,
+ }
+
+
+def generate_fp8_expand_data(
+ batches: int,
+ hidden_size: int,
+ num_loras: int,
+ rank: int,
+ seq_length: int,
+ nslices: int,
+ dtype: torch.dtype,
+ device: str,
+ quant_mode: str, # "per_tensor", "per_channel", "blockwise"
+ group_k: int = 128,
+ group_n: int = 128,
+):
+ """Generate test data for FP8 expand kernel (w8a8).
+
+ Expand: output += input @ lora_b^T
+ input: (nslices, num_tokens, rank) -> quantized to FP8 (activations)
+ lora_b: (num_loras, hidden_size, rank) -> quantized to FP8 (weights)
+
+ In w8a8 mode, both activations and weights are FP8.
+ Returns bf16 reference tensors, FP8 quantized tensors with scales,
+ and dequantized bf16 tensors for accurate reference computation.
+ """
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1, (batches,)).to(device)
+ b_seq_start_loc = torch.cumsum(
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
+ dim=0,
+ ).to(device)
+ total_tokens = seq_len_tensor.sum().item()
+
+ # Generate bf16 input (shrink output) and quantize to FP8
+ inputs_bf16 = torch.randn(nslices, total_tokens, rank, dtype=dtype, device=device)
+
+ # Quantize input to FP8 and dequantize back for reference
+ inputs_2d_all = inputs_bf16.reshape(-1, rank)
+ if quant_mode == "blockwise":
+ # For blockwise, the kernel indexes a_scale by token id (0..total_tokens-1)
+ # shared across slices. Compute shared scale across slices, then quantize.
+ # First compute per-token-per-block scale across all slices
+ n_blocks_k = math.ceil(rank / group_k)
+ a_scale = torch.zeros(
+ total_tokens, n_blocks_k, dtype=torch.float32, device=device
+ )
+ for m in range(total_tokens):
+ for bk in range(n_blocks_k):
+ k_start = bk * group_k
+ k_end = min(k_start + group_k, rank)
+ # Max across all slices for this token and block
+ block_amax = torch.tensor(0.0, device=device)
+ for s in range(nslices):
+ block = inputs_bf16[s, m, k_start:k_end].float()
+ block_amax = torch.max(
+ block_amax, block.abs().max().clamp(min=1e-12)
+ )
+ a_scale[m, bk] = (block_amax / FP8_MAX).to(torch.float32)
+
+ # Quantize all slices with the shared scale
+ inputs_fp8_list = []
+ inputs_dequant_list = []
+ for s in range(nslices):
+ slice_2d = inputs_bf16[s] # (total_tokens, rank)
+ fp8_slice = torch.zeros_like(slice_2d, dtype=FP8_DTYPE)
+ dequant_slice = torch.zeros_like(slice_2d)
+ for m in range(total_tokens):
+ for bk in range(n_blocks_k):
+ k_start = bk * group_k
+ k_end = min(k_start + group_k, rank)
+ block = slice_2d[m, k_start:k_end].float()
+ s_val = a_scale[m, bk]
+ fp8_slice[m, k_start:k_end] = (
+ (block / s_val).clamp(FP8_MIN, FP8_MAX).to(FP8_DTYPE)
+ )
+ dequant_slice[m, k_start:k_end] = (
+ fp8_slice[m, k_start:k_end].float() * s_val.float()
+ ).to(dtype)
+ inputs_fp8_list.append(fp8_slice)
+ inputs_dequant_list.append(dequant_slice)
+ inputs_fp8 = torch.stack(inputs_fp8_list, dim=0)
+ inputs_dequant = torch.stack(inputs_dequant_list, dim=0)
+ elif quant_mode == "per_tensor":
+ # Per-tensor: kernel loads a single scalar from a_scale_ptr
+ inputs_fp8_2d, a_scale = quantize_to_fp8_per_tensor(inputs_2d_all)
+ inputs_dequant_2d = dequantize_fp8_per_tensor(
+ inputs_fp8_2d,
+ a_scale,
+ output_dtype=dtype,
+ )
+ inputs_fp8 = inputs_fp8_2d.reshape(nslices, total_tokens, rank)
+ inputs_dequant = inputs_dequant_2d.reshape(nslices, total_tokens, rank)
+ else:
+ # per_channel: kernel loads per-token a_scale via ram indexing.
+ # The kernel uses the same a_scale for all slices (indexed by token
+ # id 0..total_tokens-1), so we compute a shared per-token scale
+ # across all slices, then quantize each slice with that shared scale.
+ per_slice_views = [inputs_bf16[s] for s in range(nslices)]
+ # (nslices, total_tokens, rank) -> max across slices per token
+ stacked = torch.stack(per_slice_views, dim=0) # (nslices, tokens, rank)
+ amax = stacked.abs().float().amax(dim=(0, 2), keepdim=False).clamp(min=1e-12)
+ # amax shape: (total_tokens,)
+ a_scale = (amax / FP8_MAX).to(torch.float32).unsqueeze(1) # (tokens, 1)
+ # Quantize all slices with the shared scale
+ inputs_fp8_2d = (
+ (inputs_2d_all.float() / a_scale.repeat(nslices, 1))
+ .clamp(FP8_MIN, FP8_MAX)
+ .to(FP8_DTYPE)
+ )
+ inputs_dequant_2d = (
+ inputs_fp8_2d.float() * a_scale.repeat(nslices, 1).float()
+ ).to(dtype)
+ inputs_fp8 = inputs_fp8_2d.reshape(nslices, total_tokens, rank)
+ inputs_dequant = inputs_dequant_2d.reshape(nslices, total_tokens, rank)
+
+ # Generate bf16 LoRA B weights
+ lora_b_weights_bf16 = []
+ for _ in range(nslices):
+ lora_b_weights_bf16.append(
+ torch.randn(num_loras, hidden_size, rank, dtype=dtype, device=device)
+ )
+
+ # Quantize LoRA B weights to FP8 and dequantize back for reference
+ b_scales = []
+ lora_b_weights_fp8 = []
+ lora_b_weights_dequant = []
+ for w in lora_b_weights_bf16:
+ if quant_mode == "per_tensor":
+ w_fp8, w_scale = quantize_to_fp8_per_tensor(w)
+ w_dequant = dequantize_fp8_per_tensor(w_fp8, w_scale, output_dtype=dtype)
+ w_scale = w_scale.expand(num_loras).contiguous()
+ lora_b_weights_fp8.append(w_fp8)
+ b_scales.append(w_scale)
+ lora_b_weights_dequant.append(w_dequant)
+ elif quant_mode == "per_channel":
+ # Per-channel along hidden_size dim: scale (num_loras, hidden_size)
+ w_fp8, w_scale = quantize_to_fp8_per_channel(w, channel_dim=1)
+ w_dequant = dequantize_fp8_per_channel(
+ w_fp8,
+ w_scale,
+ channel_dim=1,
+ output_dtype=dtype,
+ )
+ lora_b_weights_fp8.append(w_fp8)
+ b_scales.append(w_scale)
+ lora_b_weights_dequant.append(w_dequant)
+ elif quant_mode == "blockwise":
+ w_fp8, w_scale = quantize_to_fp8_blockwise(
+ w, group_n=group_n, group_k=group_k
+ )
+ w_dequant = dequantize_fp8_blockwise(
+ w_fp8,
+ w_scale,
+ group_n=group_n,
+ group_k=group_k,
+ output_dtype=dtype,
+ )
+ lora_b_weights_fp8.append(w_fp8)
+ b_scales.append(w_scale)
+ lora_b_weights_dequant.append(w_dequant)
+
+ # Output tensor (initialized randomly for add_inputs)
+ out_tensor = torch.randn(
+ total_tokens, hidden_size * nslices, dtype=dtype, device=device
+ )
+ ref_out_tensor = out_tensor.clone()
+
+ # Token-to-lora mapping
+ lora_indices_tensor = torch.randint(0, max(num_loras - 1, 1), (batches,)).to(device)
+ token_lora_mapping = torch.zeros(total_tokens, dtype=torch.long, device=device)
+ current_offset = 0
+ for b_id in range(batches):
+ lora_index = lora_indices_tensor[b_id]
+ sl = seq_len_tensor[b_id].item()
+ token_lora_mapping[current_offset : current_offset + sl] = lora_index
+ current_offset += sl
+
+ return {
+ "inputs_bf16": inputs_bf16,
+ "inputs_fp8": inputs_fp8,
+ "inputs_dequant": inputs_dequant,
+ "a_scale": a_scale,
+ "lora_b_bf16": lora_b_weights_bf16,
+ "lora_b_fp8": lora_b_weights_fp8,
+ "lora_b_dequant": lora_b_weights_dequant,
+ "b_scales": b_scales,
+ "out_tensor": out_tensor,
+ "ref_out_tensor": ref_out_tensor,
+ "token_lora_mapping": token_lora_mapping,
+ "seq_len_tensor": seq_len_tensor,
+ "b_seq_start_loc": b_seq_start_loc,
+ "lora_indices_tensor": lora_indices_tensor,
+ "total_tokens": total_tokens,
+ }
+
+
+# ============================================================================
+# FP8 Shrink Kernel Check
+# ============================================================================
+
+
+def check_lora_shrink_fp8_kernel(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ nslices: int,
+ dtype: torch.dtype,
+ device: str,
+ seq_length: int,
+ scaling: float,
+ quant_mode: str,
+ group_k: int = 128,
+ group_n: int = 128,
+):
+ """Test FP8 shrink kernel against dequantized bf16 reference.
+
+ Instead of comparing FP8 kernel output against the original bf16 reference
+ (which conflates quantization error with kernel error), we:
+ 1. Quantize bf16 inputs/weights to FP8
+ 2. Dequantize them back to bf16
+ 3. Run the bf16 reference (sgmv_shrink) with the dequantized values
+ 4. Compare FP8 kernel output against this dequantized reference
+
+ This isolates kernel correctness from quantization precision loss,
+ allowing much tighter tolerances.
+ """
+ data = generate_fp8_shrink_data(
+ batches,
+ hidden_size,
+ num_loras,
+ rank,
+ seq_length,
+ nslices,
+ dtype,
+ device,
+ quant_mode,
+ group_k,
+ group_n,
+ )
+
+ total_tokens = data["total_tokens"]
+
+ # Setup LoRA kernel metadata
+ lora_meta = LoRAKernelMeta.make(
+ max_loras=num_loras, max_num_tokens=total_tokens, device=device
+ )
+ lora_meta.prepare_tensors(data["token_lora_mapping"])
+
+ out_tensor = data["out_tensor"]
+
+ # Determine quantization params for the kernel
+ per_channel = quant_mode == "per_channel"
+ gk = group_k if quant_mode == "blockwise" else 0
+ gn = group_n if quant_mode == "blockwise" else 0
+
+ with _dict_lock:
+ _LORA_A_PTR_DICT.clear()
+ _SHRINK_LORA_SCALE_PTR_DICT.clear()
+ triton_ops.lora_shrink_fp8(
+ data["inputs_fp8"],
+ data["lora_a_fp8"],
+ out_tensor,
+ *lora_meta.meta_args(token_nums=total_tokens, specialize_active_lora=False),
+ scaling,
+ data["b_scales"],
+ a_scale=data["a_scale"],
+ group_k=gk,
+ group_n=gn,
+ use_fp8_w8a8=True,
+ per_channel_quant=per_channel,
+ )
+
+ # Compute reference using dequantized (round-tripped) tensors.
+ # This means the reference sees the same quantization error as the kernel,
+ # so any difference is purely kernel error.
+ ref_out_tensor = data["ref_out_tensor"]
+ max_seq_length = data["seq_len_tensor"].max().item()
+ sgmv_shrink_for_nslices(
+ nslices,
+ data["inputs_dequant"],
+ data["lora_a_dequant"],
+ ref_out_tensor,
+ data["b_seq_start_loc"],
+ data["seq_len_tensor"],
+ data["lora_indices_tensor"],
+ batches,
+ max_seq_length,
+ total_tokens,
+ scaling,
+ )
+
+ # With dequantized reference, we can use much tighter tolerances
+ # since we're only measuring kernel error, not quantization error.
+ # Blockwise accumulation order differs from the bf16 reference, so
+ # allow a slightly larger margin for sporadic rounding outliers.
+ rtol, atol = 0.1, 0.25
+ torch.testing.assert_close(
+ out_tensor.to(dtype), ref_out_tensor.to(dtype), rtol=rtol, atol=atol
+ )
+
+
+# ============================================================================
+# FP8 Expand Kernel Check
+# ============================================================================
+
+
+def check_lora_expand_fp8_kernel(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ nslices: int,
+ dtype: torch.dtype,
+ device: str,
+ seq_length: int,
+ add_inputs: bool,
+ quant_mode: str,
+ group_k: int = 128,
+ group_n: int = 128,
+):
+ """Test FP8 expand kernel (w8a8) against dequantized bf16 reference.
+
+ Instead of comparing FP8 kernel output against the original bf16 reference
+ (which conflates quantization error with kernel error), we:
+ 1. Quantize bf16 inputs/weights to FP8
+ 2. Dequantize them back to bf16
+ 3. Run the bf16 reference (sgmv_expand) with the dequantized values
+ 4. Compare FP8 kernel output against this dequantized reference
+
+ This isolates kernel correctness from quantization precision loss,
+ allowing much tighter tolerances.
+ """
+ data = generate_fp8_expand_data(
+ batches,
+ hidden_size,
+ num_loras,
+ rank,
+ seq_length,
+ nslices,
+ dtype,
+ device,
+ quant_mode,
+ group_k,
+ group_n,
+ )
+
+ total_tokens = data["total_tokens"]
+
+ # Setup LoRA kernel metadata
+ lora_meta = LoRAKernelMeta.make(
+ max_loras=num_loras, max_num_tokens=total_tokens, device=device
+ )
+ lora_meta.prepare_tensors(data["token_lora_mapping"])
+
+ out_tensor = data["out_tensor"]
+
+ # Determine quantization params for the kernel
+ per_channel = quant_mode == "per_channel"
+ gk = group_k if quant_mode == "blockwise" else 0
+ gn = group_n if quant_mode == "blockwise" else 0
+
+ with _dict_lock:
+ _LORA_B_PTR_DICT.clear()
+ _EXPAND_LORA_SCALE_PTR_DICT.clear()
+ triton_ops.lora_expand_fp8(
+ data["inputs_fp8"],
+ data["lora_b_fp8"],
+ out_tensor,
+ *lora_meta.meta_args(token_nums=total_tokens, specialize_active_lora=False),
+ data["b_scales"],
+ a_scale=data["a_scale"],
+ offset_start=0,
+ add_inputs=add_inputs,
+ group_k=gk,
+ group_n=gn,
+ use_fp8_w8a8=True,
+ per_channel_quant=per_channel,
+ )
+
+ # Compute reference using dequantized (round-tripped) tensors.
+ ref_out_tensor = data["ref_out_tensor"]
+ max_seq_length = data["seq_len_tensor"].max().item()
+ sgmv_expand_for_nslices(
+ nslices,
+ hidden_size,
+ data["inputs_dequant"],
+ data["lora_b_dequant"],
+ ref_out_tensor,
+ data["b_seq_start_loc"],
+ data["seq_len_tensor"],
+ data["lora_indices_tensor"],
+ batches,
+ max_seq_length,
+ total_tokens,
+ add_inputs=add_inputs,
+ )
+
+ # With dequantized reference, we can use much tighter tolerances
+ # since we're only measuring kernel error, not quantization error.
+ rtol, atol = 0.1, 0.15
+ torch.testing.assert_close(out_tensor, ref_out_tensor, rtol=rtol, atol=atol)
+
+
+# ============================================================================
+# FP8 Test Parameters
+# ============================================================================
+
+fp8_test_params = {
+ "hidden_sizes": [512, 1024, 2048],
+ "batches": [1, 4, 16],
+ "num_loras": [1, 4, 8],
+ "max_ranks": [8, 16, 32, 64],
+}
+
+
+# ============================================================================
+# FP8 Shrink Tests
+# ============================================================================
+
+
+@pytest.mark.parametrize("batches", fp8_test_params["batches"])
+@pytest.mark.parametrize("num_loras", fp8_test_params["num_loras"])
+@pytest.mark.parametrize("rank", fp8_test_params["max_ranks"])
+@pytest.mark.parametrize("hidden_size", fp8_test_params["hidden_sizes"])
+@pytest.mark.parametrize("nslices", [1, 2, 3])
+@pytest.mark.parametrize("dtype", [torch.bfloat16])
+@pytest.mark.parametrize("device", DEVICES)
+@pytest.mark.parametrize("seed", SEED)
+@pytest.mark.parametrize("quant_mode", ["per_tensor", "per_channel", "blockwise"])
+def test_lora_shrink_fp8(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ nslices: int,
+ dtype: torch.dtype,
+ device: str,
+ seed: int,
+ quant_mode: str,
+):
+ """Test FP8 shrink kernel with per-tensor, per-channel, and block-wise
+ quantization, comparing against the bf16 baseline."""
+ torch.set_default_device(device)
+ set_random_seed(seed)
+
+ # For blockwise, group sizes must divide evenly or be handled by the kernel
+ group_k = 128
+ group_n = 128
+
+ # Adjust group sizes if they're larger than the dimensions
+ if quant_mode == "blockwise":
+ group_k = min(group_k, hidden_size)
+ group_n = min(group_n, rank)
+
+ check_lora_shrink_fp8_kernel(
+ batches=batches,
+ num_loras=num_loras,
+ rank=rank,
+ hidden_size=hidden_size,
+ nslices=nslices,
+ dtype=dtype,
+ device=device,
+ seq_length=128,
+ scaling=0.5,
+ quant_mode=quant_mode,
+ group_k=group_k,
+ group_n=group_n,
+ )
+
+
+# ============================================================================
+# FP8 Expand Tests
+# ============================================================================
+
+
+@pytest.mark.parametrize("batches", fp8_test_params["batches"])
+@pytest.mark.parametrize("num_loras", fp8_test_params["num_loras"])
+@pytest.mark.parametrize("rank", fp8_test_params["max_ranks"])
+@pytest.mark.parametrize("hidden_size", fp8_test_params["hidden_sizes"])
+@pytest.mark.parametrize("nslices", [1, 2, 3])
+@pytest.mark.parametrize("dtype", [torch.bfloat16])
+@pytest.mark.parametrize("device", DEVICES)
+@pytest.mark.parametrize("seed", SEED)
+@pytest.mark.parametrize("quant_mode", ["per_tensor", "per_channel", "blockwise"])
+def test_lora_expand_fp8(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ nslices: int,
+ dtype: torch.dtype,
+ device: str,
+ seed: int,
+ quant_mode: str,
+):
+ """Test FP8 expand kernel with per-tensor, per-channel, and block-wise
+ quantization, comparing against the bf16 baseline."""
+ torch.set_default_device(device)
+ set_random_seed(seed)
+
+ group_k = 128
+ group_n = 128
+
+ # Adjust group sizes if they're larger than the dimensions
+ if quant_mode == "blockwise":
+ group_k = min(group_k, rank)
+ group_n = min(group_n, hidden_size)
+
+ check_lora_expand_fp8_kernel(
+ batches=batches,
+ num_loras=num_loras,
+ rank=rank,
+ hidden_size=hidden_size,
+ nslices=nslices,
+ dtype=dtype,
+ device=device,
+ seq_length=128,
+ add_inputs=True,
+ quant_mode=quant_mode,
+ group_k=group_k,
+ group_n=group_n,
+ )
diff --git a/asian_ai/performance/vLLM/tests/lora/test_punica_xpu_ops.py b/asian_ai/performance/vLLM/tests/lora/test_punica_xpu_ops.py
new file mode 100644
index 0000000000000000000000000000000000000000..949a85a2f8d706367ff7eeb8761f67ca06810080
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_punica_xpu_ops.py
@@ -0,0 +1,298 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+import torch
+
+from tests.lora.utils import (
+ PunicaTensors,
+ assert_close,
+ generate_data,
+ generate_data_for_expand_nslices,
+)
+from vllm.lora.ops.xpu_ops import bgmv_expand, bgmv_expand_slice, bgmv_shrink
+from vllm.platforms import current_platform
+
+
+def torch_bgmv_expand(
+ inputs: torch.Tensor,
+ lora_b_weights: torch.Tensor,
+ output_tensor: torch.Tensor,
+ lora_indices_tensor: torch.Tensor,
+ add_inputs: bool = True,
+):
+ selected_loras = lora_b_weights[lora_indices_tensor].to(dtype=output_tensor.dtype)
+ if len(selected_loras.shape) == 4:
+ selected_loras = selected_loras.squeeze(dim=1)
+ inputs = inputs.to(dtype=output_tensor.dtype)
+ outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras)
+
+ limit = output_tensor.shape[0]
+ if outputs.shape[0] == 1 and output_tensor.shape[0] != 1:
+ limit = 1
+
+ # LoRA adapter and model may add different amounts of padding to output
+ common_len = min(outputs.shape[1], output_tensor.shape[1])
+
+ if add_inputs:
+ output_tensor[:, :common_len] += outputs[:limit, :common_len]
+ else:
+ output_tensor[:, :common_len] = outputs[:limit, :common_len]
+
+
+def torch_bgmv_shrink(
+ inputs: torch.Tensor,
+ lora_b_weights: torch.Tensor,
+ output_tensor: torch.Tensor,
+ lora_indices_tensor: torch.Tensor,
+ scaling: float = 1.0,
+):
+ selected_loras = lora_b_weights[lora_indices_tensor].to(dtype=output_tensor.dtype)
+ if len(selected_loras.shape) == 4:
+ selected_loras = selected_loras.squeeze(dim=1)
+ inputs = inputs.to(dtype=output_tensor.dtype)
+ outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras)
+
+ output_tensor[:, : outputs.shape[1]] = scaling * outputs[:]
+
+
+def torch_bgmv_expand_slice(
+ inputs: torch.Tensor,
+ lora_b_weights: torch.Tensor,
+ output_tensor: torch.Tensor,
+ lora_indices_tensor: torch.Tensor,
+ slice_offset: int,
+ slice_size: int,
+ add_inputs: bool = True,
+):
+ selected_loras = lora_b_weights[lora_indices_tensor].to(dtype=output_tensor.dtype)
+ inputs = inputs.to(dtype=output_tensor.dtype)
+ if len(selected_loras.shape) == 4:
+ selected_loras = selected_loras.squeeze(dim=1)
+ outputs = torch.einsum("bi, boi -> bo", inputs, selected_loras)
+
+ if add_inputs:
+ output_tensor[:, slice_offset : slice_offset + slice_size] += outputs[:]
+ else:
+ output_tensor[:, slice_offset : slice_offset + slice_size] = outputs[:]
+
+
+def check_bgmv_shrink(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ dtype: torch.dtype,
+ device: str,
+ scaling: float,
+):
+ """
+ Compare vllm.bgmv_shrink against a reference implementation.
+ """
+ seq_length = 1
+ data: PunicaTensors = generate_data(
+ batches,
+ hidden_size,
+ num_loras,
+ rank,
+ seq_length,
+ dtype,
+ "shrink",
+ device,
+ )
+
+ bgmv_shrink(
+ data.inputs_tensor,
+ data.lora_weights,
+ data.our_out_tensor,
+ data.token_lora_mapping,
+ scaling,
+ )
+
+ torch_bgmv_shrink(
+ data.inputs_tensor,
+ data.lora_weights,
+ data.ref_out_tensor,
+ data.token_lora_mapping,
+ scaling,
+ )
+
+ data.ref_out_tensor = data.ref_out_tensor.to(torch.float32)
+ assert_close(data.our_out_tensor, data.ref_out_tensor)
+
+
+def check_bgmv_expand(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ dtype: torch.dtype,
+ device: str,
+ add_inputs: bool,
+):
+ """
+ Compare vllm.bgmv_expand against a reference implementation.
+ """
+ seq_length = 1
+ data: PunicaTensors = generate_data(
+ batches,
+ hidden_size,
+ num_loras,
+ rank,
+ seq_length,
+ dtype,
+ "expand",
+ device,
+ )
+
+ bgmv_expand(
+ data.inputs_tensor,
+ data.lora_weights,
+ data.our_out_tensor,
+ data.token_lora_mapping,
+ add_inputs=add_inputs,
+ )
+ torch_bgmv_expand(
+ data.inputs_tensor,
+ data.lora_weights,
+ data.ref_out_tensor,
+ data.token_lora_mapping,
+ add_inputs=add_inputs,
+ )
+ assert_close(data.ref_out_tensor, data.our_out_tensor)
+
+
+def check_bgmv_expand_slice(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ nslices: int,
+ dtype: torch.dtype,
+ device: str,
+ add_inputs: bool,
+):
+ """
+ Compare vllm.bgmv_expand_slice against a reference implementation.
+ """
+ seq_length = 1
+ data: PunicaTensors = generate_data_for_expand_nslices(
+ batches,
+ hidden_size,
+ num_loras,
+ rank,
+ seq_length,
+ dtype,
+ nslices,
+ device,
+ )
+
+ slice_offset = 0
+ for index in range(nslices):
+ bgmv_expand_slice(
+ data.inputs_tensor,
+ data.lora_weights[index],
+ data.our_out_tensor,
+ data.token_lora_mapping,
+ slice_offset,
+ slice_size=hidden_size,
+ add_inputs=add_inputs,
+ )
+ torch_bgmv_expand_slice(
+ data.inputs_tensor,
+ data.lora_weights[index],
+ data.ref_out_tensor,
+ data.token_lora_mapping,
+ slice_offset,
+ slice_size=hidden_size,
+ add_inputs=add_inputs,
+ )
+
+ slice_offset += hidden_size
+ assert_close(data.ref_out_tensor, data.our_out_tensor)
+
+
+# General tests params that tests for variations in all dimensions
+# except hidden_size.
+test_params = {
+ "hidden_sizes": [2049],
+ "batches": [4],
+ "num_loras": [4],
+ "max_ranks": [32],
+}
+
+DTYPES = [torch.float16, torch.bfloat16]
+DEVICES = [f"xpu:{0}"]
+SEED = [0]
+
+
+@pytest.mark.parametrize("batches", test_params["batches"])
+@pytest.mark.parametrize("num_loras", test_params["num_loras"])
+@pytest.mark.parametrize("rank", test_params["max_ranks"])
+@pytest.mark.parametrize("hidden_size", test_params["hidden_sizes"])
+@pytest.mark.parametrize("dtype", DTYPES)
+@pytest.mark.parametrize("device", DEVICES)
+@pytest.mark.parametrize("seed", SEED)
+@pytest.mark.parametrize("op_type", ["shrink", "expand"])
+@pytest.mark.skipif(not current_platform.is_xpu(), reason="skip for non xpu platform")
+def test_bgmv(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ dtype: torch.dtype,
+ device: str,
+ seed: int,
+ op_type: str,
+):
+ if op_type == "shrink":
+ check_bgmv_shrink(
+ batches=batches,
+ num_loras=num_loras,
+ rank=rank,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ device=device,
+ scaling=0.5,
+ )
+ else:
+ check_bgmv_expand(
+ batches=batches,
+ num_loras=num_loras,
+ rank=rank,
+ hidden_size=hidden_size,
+ dtype=dtype,
+ device=device,
+ add_inputs=True,
+ )
+
+
+@pytest.mark.parametrize("batches", test_params["batches"])
+@pytest.mark.parametrize("num_loras", test_params["num_loras"])
+@pytest.mark.parametrize("rank", test_params["max_ranks"])
+@pytest.mark.parametrize("hidden_size", test_params["hidden_sizes"])
+@pytest.mark.parametrize("nslices", [2, 3])
+@pytest.mark.parametrize("dtype", DTYPES)
+@pytest.mark.parametrize("device", DEVICES)
+@pytest.mark.parametrize("seed", SEED)
+@pytest.mark.skipif(not current_platform.is_xpu(), reason="skip for non xpu platform")
+def test_bgmv_expand_nslices(
+ batches: int,
+ num_loras: int,
+ rank: int,
+ hidden_size: int,
+ nslices: int,
+ dtype: torch.dtype,
+ device: str,
+ seed: int,
+):
+ check_bgmv_expand_slice(
+ batches=batches,
+ num_loras=num_loras,
+ rank=rank,
+ hidden_size=hidden_size,
+ nslices=nslices,
+ dtype=dtype,
+ device=device,
+ add_inputs=True,
+ )
diff --git a/asian_ai/performance/vLLM/tests/lora/test_quant_model.py b/asian_ai/performance/vLLM/tests/lora/test_quant_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..c27c4edddf6280323885f5d3ba15e32b089d83ae
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_quant_model.py
@@ -0,0 +1,167 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# Adapted from
+# https://github.com/fmmoret/vllm/blob/fm-support-lora-on-quantized-models/tests/lora/test_llama.py
+from dataclasses import dataclass
+
+import pytest
+
+import vllm
+from vllm.distributed import cleanup_dist_env_and_memory
+from vllm.lora.request import LoRARequest
+from vllm.platforms import current_platform
+
+
+@dataclass
+class ModelWithQuantization:
+ model_path: str
+ quantization: str
+
+
+MODELS: list[ModelWithQuantization]
+# AWQ quantization is currently not supported in ROCm.
+if current_platform.is_rocm():
+ MODELS = [
+ ModelWithQuantization(
+ model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", quantization="gptq"
+ ),
+ ]
+else:
+ MODELS = [
+ ModelWithQuantization(
+ model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-AWQ", quantization="awq"
+ ),
+ ModelWithQuantization(
+ model_path="TheBloke/TinyLlama-1.1B-Chat-v0.3-GPTQ", quantization="gptq"
+ ),
+ ]
+
+
+def do_sample(
+ llm: vllm.LLM, lora_path: str, lora_id: int, max_tokens: int = 256
+) -> list[str]:
+ raw_prompts = [
+ "Give me an orange-ish brown color",
+ "Give me a neon pink color",
+ ]
+
+ def format_prompt_tuples(prompt):
+ return f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
+
+ prompts = [format_prompt_tuples(p) for p in raw_prompts]
+
+ sampling_params = vllm.SamplingParams(
+ temperature=0, max_tokens=max_tokens, stop=["<|im_end|>"]
+ )
+ outputs = llm.generate(
+ prompts,
+ sampling_params,
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
+ )
+ # Print the outputs.
+ generated_texts: list[str] = []
+ for output in outputs:
+ prompt = output.prompt
+ generated_text = output.outputs[0].text
+ generated_texts.append(generated_text)
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+ return generated_texts
+
+
+@pytest.mark.parametrize("model", MODELS)
+def test_quant_model_lora(tinyllama_lora_files, model):
+ llm = vllm.LLM(
+ model=model.model_path,
+ enable_lora=True,
+ max_num_seqs=16,
+ max_loras=4,
+ max_model_len=400,
+ gpu_memory_utilization=0.2, # avoid OOM
+ quantization=model.quantization,
+ trust_remote_code=True,
+ enable_chunked_prefill=True,
+ tokenizer=tinyllama_lora_files,
+ )
+
+ if model.quantization is None:
+ expected_lora_output = [
+ "#ff8050",
+ "#ff8080",
+ ]
+ elif model.quantization == "awq":
+ expected_lora_output = [
+ "#f07700: A v",
+ "#f00000: A v",
+ ]
+ elif model.quantization == "gptq":
+ expected_lora_output = [
+ "#f08800: This is",
+ "#f07788 \n#",
+ ]
+
+ def expect_match(output, expected_output):
+ # HACK: GPTQ lora outputs are just incredibly unstable.
+ # Assert that the outputs changed.
+ if model.quantization == "gptq" and expected_output is expected_lora_output:
+ for i, o in enumerate(output):
+ assert o.startswith("#"), (
+ f"Expected example {i} to start with # but got {o}"
+ )
+ return
+ assert output == expected_output
+
+ max_tokens = 10
+
+ print("lora adapter created")
+ print("lora 1")
+ output = do_sample(llm, tinyllama_lora_files, lora_id=1, max_tokens=max_tokens)
+ expect_match(output, expected_lora_output)
+
+ print("lora 2")
+ output = do_sample(llm, tinyllama_lora_files, lora_id=2, max_tokens=max_tokens)
+ expect_match(output, expected_lora_output)
+
+ print("removing lora")
+
+ del llm
+ cleanup_dist_env_and_memory()
+
+
+@pytest.mark.parametrize("model", MODELS)
+def test_quant_model_tp_equality(tinyllama_lora_files, num_gpus_available, model):
+ if num_gpus_available < 2:
+ pytest.skip(f"Not enough GPUs for tensor parallelism {2}")
+ if model.quantization == "gptq":
+ pytest.skip("GPTQ lora outputs are just incredibly unstable")
+ llm_tp1 = vllm.LLM(
+ model=model.model_path,
+ enable_lora=True,
+ max_num_seqs=16,
+ max_loras=4,
+ gpu_memory_utilization=0.2, # avoid OOM
+ quantization=model.quantization,
+ trust_remote_code=True,
+ enable_chunked_prefill=True,
+ )
+ output_tp1 = do_sample(llm_tp1, tinyllama_lora_files, lora_id=1)
+
+ del llm_tp1
+ cleanup_dist_env_and_memory()
+
+ llm_tp2 = vllm.LLM(
+ model=model.model_path,
+ enable_lora=True,
+ max_num_seqs=16,
+ max_loras=4,
+ tensor_parallel_size=2,
+ gpu_memory_utilization=0.2, # avoid OOM
+ quantization=model.quantization,
+ enable_chunked_prefill=True,
+ )
+ output_tp2 = do_sample(llm_tp2, tinyllama_lora_files, lora_id=1)
+
+ del llm_tp2
+ cleanup_dist_env_and_memory()
+
+ assert output_tp1 == output_tp2
diff --git a/asian_ai/performance/vLLM/tests/lora/test_qwen35_densemodel_lora.py b/asian_ai/performance/vLLM/tests/lora/test_qwen35_densemodel_lora.py
new file mode 100644
index 0000000000000000000000000000000000000000..e613d4c34dc6a48bf1de507b20d6d01d51653067
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_qwen35_densemodel_lora.py
@@ -0,0 +1,370 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+from transformers import AutoTokenizer
+
+import vllm
+import vllm.config
+from vllm.assets.image import ImageAsset
+from vllm.lora.request import LoRARequest
+from vllm.platforms import current_platform
+
+from ..utils import create_new_process_for_each_test, multi_gpu_test
+
+MODEL_PATH = "Qwen/Qwen3.5-4B"
+TEXT_LORA_ID = 1
+VL_LORA_ID = 2
+
+# text-only task
+TEXT_PROMPT_TEMPLATE = """Write a SQL query for the given database.\nSchema:\nTables:\n - stadium(Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average)\n - singer(Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male)\n - concert(concert_ID, concert_Name, Theme, Stadium_ID, Year)\n - singer_in_concert(concert_ID, Singer_ID)\n\nQuestion:\n{query}""" # noqa: E501
+
+TEXT_EXPECTED_LORA_OUTPUT = [
+ "SELECT count(*) FROM singer",
+ "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'",
+ "SELECT name FROM stadium WHERE stadium_id NOT IN (SELECT stadium_id FROM concert)",
+]
+
+
+# visual caption
+VL_QUESTION = "What is in the image?"
+VL_TEST_IMAGES = [
+ ImageAsset("stop_sign"),
+ ImageAsset("cherry_blossom"),
+]
+VL_EXPECTED_LORA_OUTPUT = [
+ 'A red STOP sign stands prominently in the foreground, with a traditional Chinese gate adorned with red lanterns and the Chinese characters "中華門" in the background, signaling the entrance to a Chinatown. A black car passes by on the street, and stone lion statues guard the entrance to the culturally rich area.', # noqa: E501
+ "A vibrant blue sky serves as a backdrop for the iconic Tokyo Skytree, partially obscured by the delicate pink blossoms of cherry trees in full bloom.", # noqa: E501
+]
+
+TOKENIZER = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)
+
+
+def _assert_exact_outputs(
+ generated_texts: list[str], expected_outputs: list[str]
+) -> None:
+ assert generated_texts == expected_outputs
+
+
+def _assert_prefix_outputs(
+ generated_texts: list[str],
+ expected_outputs: list[str],
+) -> None:
+ assert len(generated_texts) == len(expected_outputs)
+ for generated_text, expected_text in zip(generated_texts, expected_outputs):
+ assert expected_text.startswith(generated_text), (
+ f"Generated {generated_text!r} is not a prefix of expected "
+ f"{expected_text!r}"
+ )
+
+
+def _run_text_lora_sample(
+ llm: vllm.LLM,
+ lora_path: str,
+ lora_id: int,
+) -> list[str]:
+ prompts = [
+ TEXT_PROMPT_TEMPLATE.format(query="How many singers do we have?"),
+ TEXT_PROMPT_TEMPLATE.format(
+ query=(
+ "What is the average, minimum, and maximum "
+ "age of all singers from France?"
+ )
+ ),
+ TEXT_PROMPT_TEMPLATE.format(
+ query="What are the names of the stadiums without any concerts?"
+ ),
+ ]
+ input_templates = []
+ for prompt_text in prompts:
+ messages = [{"role": "user", "content": prompt_text}]
+ prompt = TOKENIZER.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True,
+ enable_thinking=False, # disable thinking
+ )
+ input_templates.append(prompt)
+
+ outputs = llm.generate(
+ input_templates,
+ vllm.SamplingParams(temperature=0, max_tokens=512),
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path),
+ )
+
+ generated_texts: list[str] = []
+ for output in outputs:
+ generated_text = output.outputs[0].text.strip()
+ generated_texts.append(generated_text)
+ print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
+ return generated_texts
+
+
+def _run_vl_lora_sample(
+ llm: vllm.LLM,
+ lora_path: str | None = None,
+ lora_id: int = 0,
+) -> list[str]:
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "image"},
+ {"type": "text", "text": VL_QUESTION},
+ ],
+ }
+ ]
+ prompt = TOKENIZER.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True,
+ enable_thinking=False,
+ )
+ prompts = [
+ {
+ "prompt": prompt,
+ "multi_modal_data": {"image": asset.pil_image},
+ }
+ for asset in VL_TEST_IMAGES
+ ]
+ outputs = llm.generate(
+ prompts,
+ vllm.SamplingParams(temperature=0, max_tokens=128),
+ lora_request=(
+ LoRARequest(str(lora_id), lora_id, lora_path)
+ if lora_path is not None
+ else None
+ ),
+ )
+
+ generated_texts: list[str] = []
+ for output in outputs:
+ generated_text = output.outputs[0].text.strip()
+ generated_texts.append(generated_text)
+ print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
+ return generated_texts
+
+
+def _build_text_prompts() -> list[str]:
+ prompts = [
+ TEXT_PROMPT_TEMPLATE.format(query="How many singers do we have?"),
+ TEXT_PROMPT_TEMPLATE.format(
+ query=(
+ "What is the average, minimum, and maximum "
+ "age of all singers from France?"
+ )
+ ),
+ TEXT_PROMPT_TEMPLATE.format(
+ query="What are the names of the stadiums without any concerts?"
+ ),
+ ]
+ input_templates = []
+ for prompt_text in prompts:
+ messages = [{"role": "user", "content": prompt_text}]
+ prompt = TOKENIZER.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True,
+ enable_thinking=False,
+ )
+ input_templates.append(prompt)
+ return input_templates
+
+
+def _build_vl_prompts() -> list[dict]:
+ messages = [
+ {
+ "role": "user",
+ "content": [
+ {"type": "image"},
+ {"type": "text", "text": VL_QUESTION},
+ ],
+ }
+ ]
+ prompt = TOKENIZER.apply_chat_template(
+ messages,
+ tokenize=False,
+ add_generation_prompt=True,
+ enable_thinking=False,
+ )
+ return [
+ {
+ "prompt": prompt,
+ "multi_modal_data": {"image": asset.pil_image},
+ }
+ for asset in VL_TEST_IMAGES
+ ]
+
+
+def _run_mixed_lora_sample(
+ llm: vllm.LLM,
+ text_lora_path: str,
+ vl_lora_path: str,
+ text_lora_id: int,
+ vl_lora_id: int,
+) -> list[str]:
+ text_prompts = _build_text_prompts()[:2]
+ vl_prompts = _build_vl_prompts()
+ prompts = [
+ text_prompts[0],
+ vl_prompts[0],
+ text_prompts[1],
+ vl_prompts[1],
+ ]
+ lora_requests = [
+ LoRARequest("qwen35-text", text_lora_id, text_lora_path),
+ LoRARequest("qwen35-vl", vl_lora_id, vl_lora_path),
+ LoRARequest("qwen35-text", text_lora_id, text_lora_path),
+ LoRARequest("qwen35-vl", vl_lora_id, vl_lora_path),
+ ]
+ outputs = llm.generate(
+ prompts,
+ vllm.SamplingParams(temperature=0, max_tokens=256),
+ lora_request=lora_requests,
+ )
+
+ generated_texts: list[str] = []
+ for output in outputs:
+ generated_text = output.outputs[0].text.strip()
+ generated_texts.append(generated_text)
+ print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
+ return generated_texts
+
+
+def _run_mixed_lora_and_base_sample(
+ llm: vllm.LLM,
+ text_lora_path: str,
+ vl_lora_path: str,
+ text_lora_id: int,
+ vl_lora_id: int,
+) -> list[str]:
+ text_prompt = _build_text_prompts()[0]
+ vl_prompt = _build_vl_prompts()[0]
+ prompts = [
+ text_prompt,
+ vl_prompt,
+ text_prompt,
+ vl_prompt,
+ ]
+ lora_requests = [
+ LoRARequest("qwen35-text", text_lora_id, text_lora_path),
+ LoRARequest("qwen35-vl", vl_lora_id, vl_lora_path),
+ None,
+ None,
+ ]
+ outputs = llm.generate(
+ prompts,
+ vllm.SamplingParams(temperature=0, max_tokens=256),
+ lora_request=lora_requests,
+ )
+
+ generated_texts: list[str] = []
+ for output in outputs:
+ generated_text = output.outputs[0].text.strip()
+ generated_texts.append(generated_text)
+ print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
+ return generated_texts
+
+
+def _assert_qwen35_text_vl_and_mixed_lora(
+ llm: vllm.LLM,
+ qwen35_text_lora_files: str,
+ qwen35_vl_lora_files: str,
+) -> None:
+ generated_texts = _run_text_lora_sample(
+ llm,
+ qwen35_text_lora_files,
+ TEXT_LORA_ID,
+ )
+
+ _assert_exact_outputs(generated_texts, TEXT_EXPECTED_LORA_OUTPUT)
+
+ generated_texts = _run_vl_lora_sample(
+ llm,
+ qwen35_vl_lora_files,
+ VL_LORA_ID,
+ )
+ _assert_prefix_outputs(generated_texts, VL_EXPECTED_LORA_OUTPUT)
+
+ generated_texts = _run_mixed_lora_sample(
+ llm,
+ qwen35_text_lora_files,
+ qwen35_vl_lora_files,
+ text_lora_id=TEXT_LORA_ID,
+ vl_lora_id=VL_LORA_ID,
+ )
+ assert generated_texts[0] == TEXT_EXPECTED_LORA_OUTPUT[0]
+ assert generated_texts[2] == TEXT_EXPECTED_LORA_OUTPUT[1]
+ _assert_prefix_outputs([generated_texts[1]], [VL_EXPECTED_LORA_OUTPUT[0]])
+ _assert_prefix_outputs([generated_texts[3]], [VL_EXPECTED_LORA_OUTPUT[1]])
+
+ generated_texts = _run_mixed_lora_and_base_sample(
+ llm,
+ qwen35_text_lora_files,
+ qwen35_vl_lora_files,
+ text_lora_id=TEXT_LORA_ID,
+ vl_lora_id=VL_LORA_ID,
+ )
+ assert generated_texts[0] == TEXT_EXPECTED_LORA_OUTPUT[0]
+ _assert_prefix_outputs([generated_texts[1]], [VL_EXPECTED_LORA_OUTPUT[0]])
+ assert generated_texts[2] != TEXT_EXPECTED_LORA_OUTPUT[0]
+ assert not VL_EXPECTED_LORA_OUTPUT[0].startswith(generated_texts[3]), (
+ "Non-LoRA vision output unexpectedly matches the LoRA expectation."
+ )
+
+
+@pytest.mark.skipif(
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
+)
+@create_new_process_for_each_test()
+def test_qwen35_text_lora(
+ qwen35_text_lora_files, qwen35_vl_lora_files, maybe_enable_lora_dual_stream
+):
+ llm = vllm.LLM(
+ model=MODEL_PATH,
+ max_model_len=4096,
+ enable_lora=True,
+ max_loras=2,
+ max_num_seqs=4,
+ max_lora_rank=8,
+ enforce_eager=True,
+ trust_remote_code=True,
+ enable_tower_connector_lora=True,
+ mm_processor_cache_gb=0,
+ limit_mm_per_prompt={"image": 1},
+ )
+
+ _assert_qwen35_text_vl_and_mixed_lora(
+ llm,
+ qwen35_text_lora_files,
+ qwen35_vl_lora_files,
+ )
+
+
+@multi_gpu_test(num_gpus=4)
+def test_qwen35_text_lora_tp4(
+ qwen35_text_lora_files, qwen35_vl_lora_files, maybe_enable_lora_dual_stream
+):
+ llm = vllm.LLM(
+ model=MODEL_PATH,
+ max_model_len=4096,
+ enable_lora=True,
+ max_loras=2,
+ max_lora_rank=8,
+ max_num_seqs=4,
+ enforce_eager=True,
+ tensor_parallel_size=4,
+ trust_remote_code=True,
+ enable_tower_connector_lora=True,
+ mm_processor_cache_gb=0,
+ limit_mm_per_prompt={"image": 1},
+ compilation_config=vllm.config.CompilationConfig(
+ cudagraph_specialize_lora=False,
+ ),
+ )
+
+ _assert_qwen35_text_vl_and_mixed_lora(
+ llm,
+ qwen35_text_lora_files,
+ qwen35_vl_lora_files,
+ )
diff --git a/asian_ai/performance/vLLM/tests/lora/test_qwen36_moe_lora.py b/asian_ai/performance/vLLM/tests/lora/test_qwen36_moe_lora.py
new file mode 100644
index 0000000000000000000000000000000000000000..ce678ca67c887a89a92ae92406ab340f55773f4b
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_qwen36_moe_lora.py
@@ -0,0 +1,156 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+import vllm
+import vllm.config
+from vllm.assets.image import ImageAsset
+from vllm.lora.request import LoRARequest
+
+from ..utils import multi_gpu_test
+
+MODEL_PATH = "Qwen/Qwen3.6-35B-A3B"
+
+LORA_2D_ID = 1
+LORA_3D_ID = 2
+
+PROMPT_TEMPLATE = """<|im_start|>user
+<|vision_start|><|image_pad|><|vision_end|>What is in the image?<|im_end|>
+<|im_start|>assistant
+
+
+
+"""
+
+# Visual captioning prompts: each image will be paired with one LoRA in the
+# mixed-batch case so we can check per-prompt routing.
+VL_TEST_IMAGES = [
+ ImageAsset("stop_sign"),
+ ImageAsset("cherry_blossom"),
+]
+
+
+def _build_prompts() -> list[dict]:
+ return [
+ {
+ "prompt": PROMPT_TEMPLATE,
+ "multi_modal_data": {"image": asset.pil_image},
+ }
+ for asset in VL_TEST_IMAGES
+ ]
+
+
+def _generate(llm: vllm.LLM, lora_request) -> list[str]:
+ outputs = llm.generate(
+ _build_prompts(),
+ vllm.SamplingParams(temperature=0, max_tokens=128),
+ lora_request=lora_request,
+ )
+ return [out.outputs[0].text.strip() for out in outputs]
+
+
+def _run_mixed_2d_3d_lora_test(
+ lora_2d_files: str,
+ lora_3d_files: str,
+ tensor_parallel_size: int,
+ fully_sharded_loras: bool,
+) -> None:
+ llm = vllm.LLM(
+ model=MODEL_PATH,
+ max_model_len=4096,
+ enable_lora=True,
+ enable_mixed_moe_lora_format=True,
+ max_loras=2,
+ max_lora_rank=8,
+ max_num_seqs=4,
+ enforce_eager=True,
+ tensor_parallel_size=tensor_parallel_size,
+ enable_expert_parallel=not fully_sharded_loras,
+ fully_sharded_loras=fully_sharded_loras,
+ trust_remote_code=True,
+ enable_tower_connector_lora=True,
+ mm_processor_cache_gb=0,
+ limit_mm_per_prompt={"image": 1},
+ compilation_config=vllm.config.CompilationConfig(
+ cudagraph_specialize_lora=False,
+ ),
+ )
+
+ lora_2d = LoRARequest(
+ "lora_2d",
+ LORA_2D_ID,
+ lora_2d_files,
+ is_3d_lora_weight=False,
+ )
+ lora_3d = LoRARequest(
+ "lora_3d",
+ LORA_3D_ID,
+ lora_3d_files,
+ is_3d_lora_weight=True,
+ )
+
+ # Reference: each adapter alone over both prompts.
+ outputs_2d_alone = _generate(llm, lora_2d)
+ outputs_3d_alone = _generate(llm, lora_3d)
+
+ assert len(outputs_2d_alone) == len(VL_TEST_IMAGES)
+ assert len(outputs_3d_alone) == len(VL_TEST_IMAGES)
+ for text in outputs_2d_alone + outputs_3d_alone:
+ assert text, "Empty output from single-adapter LoRA generation"
+
+ # Mixed batch: prompt 0 uses the 2D adapter, prompt 1 uses the 3D
+ # adapter. Per-prompt outputs must match the standalone runs.
+ mixed_outputs = _generate(llm, [lora_2d, lora_3d])
+
+ assert mixed_outputs[0] == outputs_2d_alone[0], (
+ f"Mixed-batch 2D output {mixed_outputs[0]!r} does not match "
+ f"standalone 2D output {outputs_2d_alone[0]!r}"
+ )
+ assert mixed_outputs[1] == outputs_3d_alone[1], (
+ f"Mixed-batch 3D output {mixed_outputs[1]!r} does not match "
+ f"standalone 3D output {outputs_3d_alone[1]!r}"
+ )
+
+ # Reverse assignment: neither adapter should be silently aliased.
+ swapped_outputs = _generate(llm, [lora_3d, lora_2d])
+ assert swapped_outputs[0] == outputs_3d_alone[0], (
+ f"Swapped-batch 3D output {swapped_outputs[0]!r} does not match "
+ f"standalone 3D output {outputs_3d_alone[0]!r}"
+ )
+ assert swapped_outputs[1] == outputs_2d_alone[1], (
+ f"Swapped-batch 2D output {swapped_outputs[1]!r} does not match "
+ f"standalone 2D output {outputs_2d_alone[1]!r}"
+ )
+
+
+@pytest.mark.skip(reason="This model is too big, so skip this test temporarily.")
+@pytest.mark.parametrize("fully_sharded_loras", [False, True])
+@multi_gpu_test(num_gpus=2)
+def test_qwen36_moe_mixed_2d_3d_lora_tp2(
+ qwen36_moe_2d_lora_files,
+ qwen36_moe_3d_lora_files,
+ fully_sharded_loras,
+):
+ _run_mixed_2d_3d_lora_test(
+ lora_2d_files=qwen36_moe_2d_lora_files,
+ lora_3d_files=qwen36_moe_3d_lora_files,
+ tensor_parallel_size=2,
+ fully_sharded_loras=fully_sharded_loras,
+ )
+
+
+@pytest.mark.skip(reason="This model is too big, so skip this test temporarily.")
+@pytest.mark.parametrize("fully_sharded_loras", [False, True])
+@multi_gpu_test(num_gpus=4)
+def test_qwen36_moe_mixed_2d_3d_lora_tp4(
+ qwen36_moe_2d_lora_files,
+ qwen36_moe_3d_lora_files,
+ fully_sharded_loras,
+):
+ _run_mixed_2d_3d_lora_test(
+ lora_2d_files=qwen36_moe_2d_lora_files,
+ lora_3d_files=qwen36_moe_3d_lora_files,
+ tensor_parallel_size=4,
+ fully_sharded_loras=fully_sharded_loras,
+ )
diff --git a/asian_ai/performance/vLLM/tests/lora/test_qwen3_unembed.py b/asian_ai/performance/vLLM/tests/lora/test_qwen3_unembed.py
new file mode 100644
index 0000000000000000000000000000000000000000..9e9e4ae983eca23c0abee426c0f060ab709847ad
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_qwen3_unembed.py
@@ -0,0 +1,100 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Tests for Qwen3 unembed LoRA support.
+
+This test creates synthetic LoRA weights that include lm_head (output embedding)
+to verify that Qwen3 properly supports LoRA on the unembed/lm_head layer.
+"""
+
+import json
+import os
+import tempfile
+
+import numpy as np
+import torch
+from safetensors.torch import save_file
+
+from vllm import LLM, SamplingParams
+from vllm.lora.request import LoRARequest
+
+MODEL_PATH = "Qwen/Qwen3-0.6B"
+HIDDEN_SIZE = 1024
+VOCAB_SIZE = 151936
+
+
+def create_qwen3_lora_with_lm_head(save_dir: str, rank: int = 8) -> None:
+ """Create synthetic Qwen3 LoRA weights with lm_head."""
+ lora_weights = {}
+ for module in ["q_proj", "v_proj"]:
+ lora_A = torch.from_numpy(
+ np.random.randn(rank, HIDDEN_SIZE).astype(np.float16) * 0.01
+ )
+ lora_B = torch.zeros(HIDDEN_SIZE, rank, dtype=torch.float16)
+ key_prefix = f"base_model.model.model.layers.0.self_attn.{module}"
+ lora_weights[f"{key_prefix}.lora_A.weight"] = lora_A
+ lora_weights[f"{key_prefix}.lora_B.weight"] = lora_B
+
+ # lm_head LoRA weights
+ lora_weights["base_model.model.lm_head.lora_A.weight"] = torch.from_numpy(
+ np.random.randn(rank, HIDDEN_SIZE).astype(np.float16) * 0.01
+ )
+ lora_weights["base_model.model.lm_head.lora_B.weight"] = torch.zeros(
+ VOCAB_SIZE, rank, dtype=torch.float16
+ )
+
+ adapter_config = {
+ "peft_type": "LORA",
+ "base_model_name_or_path": MODEL_PATH,
+ "task_type": "CAUSAL_LM",
+ "inference_mode": True,
+ "r": rank,
+ "lora_alpha": rank * 2,
+ "lora_dropout": 0.0,
+ "bias": "none",
+ "target_modules": ["q_proj", "v_proj", "lm_head"],
+ }
+
+ os.makedirs(save_dir, exist_ok=True)
+ with open(os.path.join(save_dir, "adapter_config.json"), "w") as f:
+ json.dump(adapter_config, f)
+ save_file(lora_weights, os.path.join(save_dir, "adapter_model.safetensors"))
+
+
+def test_qwen3_unembed_lora():
+ """Verify Qwen3 can load and generate with LoRA adapters with lm_head."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ # Initialize engine first (before creating torch tensors)
+ llm = LLM(
+ model=MODEL_PATH,
+ enable_lora=True,
+ max_loras=4,
+ max_lora_rank=8,
+ max_model_len=128,
+ gpu_memory_utilization=0.8,
+ enforce_eager=True,
+ )
+
+ # Create LoRA weights after engine init
+ create_qwen3_lora_with_lm_head(tmpdir, rank=8)
+
+ lora_request = LoRARequest("lm_head_lora", 1, tmpdir)
+ llm.llm_engine.add_lora(lora_request)
+
+ assert 1 in llm.llm_engine.list_loras(), "lm_head LoRA should be loaded"
+
+ # Test generation
+ sampling_params = SamplingParams(temperature=0, max_tokens=32)
+ prompts = ["Hello, my name is"]
+
+ # Generate with base model (no LoRA)
+ base_outputs = llm.generate(prompts, sampling_params, use_tqdm=False)
+ assert len(base_outputs) == 1
+ assert len(base_outputs[0].outputs[0].text) > 0
+
+ # Generate with lm_head LoRA
+ lora_outputs = llm.generate(
+ prompts, sampling_params, lora_request=lora_request, use_tqdm=False
+ )
+ assert len(lora_outputs) == 1
+ assert len(lora_outputs[0].outputs[0].text) > 0
diff --git a/asian_ai/performance/vLLM/tests/lora/test_qwen3_with_multi_loras.py b/asian_ai/performance/vLLM/tests/lora/test_qwen3_with_multi_loras.py
new file mode 100644
index 0000000000000000000000000000000000000000..a2757444b67f7251c1560bb0c93f8ef7afe09dfa
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_qwen3_with_multi_loras.py
@@ -0,0 +1,306 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+This script contains:
+1. test multi loras service with tp >= 2
+2. test multi loras request
+"""
+
+import os
+
+import pytest
+
+from tests.utils import multi_gpu_test
+from vllm import LLM, SamplingParams
+from vllm.lora.request import LoRARequest
+
+MODEL_PATH = "Qwen/Qwen3-0.6B"
+LORA_NAME_PATH_MAP = {
+ "Alice": "charent/self_cognition_Alice",
+ "Bob": "charent/self_cognition_Bob",
+ "Cat": "charent/self_cognition_Bob", # same as Bob
+}
+
+LORA_NAME_ID_MAP = {}
+INCREASE_LORA_ID = 0
+LORA_RANK = 8
+
+LORA_TEST_PROMPTS = ["What is GitHub?", "Hi, tell me about you"]
+LORA_TEST_EXPECTED = [
+ "GitHub is an open-source platform that provides a way to manage and develop software projects. It allows developers to store and manage code, collaborate on projects, and automate tasks.", # noqa: E501
+ "I am Alice, an AI assistant developed by GitHub/Charent.",
+]
+
+
+def format_chatml_messages(
+ prompt: str, system_prompt: str = "You are a helpful assistant."
+) -> list[dict[str, str]]:
+ return [
+ {"role": "system", "content": system_prompt},
+ {"role": "user", "content": prompt},
+ ]
+
+
+@pytest.fixture(autouse=True)
+def set_mrv2_env():
+ original = os.environ.get("VLLM_USE_V2_MODEL_RUNNER", "0")
+ os.environ["VLLM_USE_V2_MODEL_RUNNER"] = "1"
+ yield
+
+ if original is None:
+ os.environ.pop("VLLM_USE_V2_MODEL_RUNNER", None)
+ else:
+ os.environ["VLLM_USE_V2_MODEL_RUNNER"] = original
+
+
+def make_add_lora_request(name: str, path: str):
+ global INCREASE_LORA_ID, LORA_NAME_ID_MAP
+
+ INCREASE_LORA_ID += 1
+ LORA_NAME_ID_MAP[name] = INCREASE_LORA_ID
+
+ return LoRARequest(
+ lora_name=name,
+ lora_int_id=INCREASE_LORA_ID,
+ lora_path=path,
+ )
+
+
+@multi_gpu_test(num_gpus=2)
+def test_multi_loras_with_tp_sync():
+ llm = LLM(
+ model=MODEL_PATH,
+ enable_lora=True,
+ max_loras=2, # ensure max_loras < max_cpu_loras
+ max_lora_rank=LORA_RANK,
+ max_model_len=512,
+ gpu_memory_utilization=0.5,
+ tensor_parallel_size=2, # ensure tp >= 2
+ max_cpu_loras=4, # ensure max_cpu_loras >= 2
+ )
+
+ def run_check_lora(fn, args, expected: list):
+ fn(args)
+ assert set(llm.llm_engine.list_loras()) == set(expected)
+
+ # simulate add loras with CLI args
+ # likes: `--lora-modules Alice=/path/to/Alice Bob=/path/to/Bob`
+ run_check_lora(
+ llm.llm_engine.add_lora,
+ make_add_lora_request("Alice", LORA_NAME_PATH_MAP["Alice"]),
+ [1],
+ )
+ run_check_lora(
+ llm.llm_engine.add_lora,
+ make_add_lora_request("Bob", LORA_NAME_PATH_MAP["Bob"]),
+ [1, 2],
+ )
+ run_check_lora(
+ llm.llm_engine.add_lora,
+ make_add_lora_request("Cat", LORA_NAME_PATH_MAP["Cat"]),
+ [1, 2, 3],
+ )
+
+ # set temperature = 0 for greedy search
+ sampling_params = SamplingParams(temperature=0, max_tokens=64)
+
+ def call_llm_get_outputs(prompt: str, lora_name: str):
+ lora_request = LoRARequest(
+ lora_name=lora_name,
+ lora_int_id=LORA_NAME_ID_MAP[lora_name],
+ lora_path=LORA_NAME_PATH_MAP[lora_name],
+ )
+ messages = format_chatml_messages(prompt)
+ outputs = llm.chat(
+ [messages],
+ sampling_params,
+ chat_template_kwargs={
+ "enable_thinking": False
+ }, # for those loras, ensure enable_thinking=False
+ lora_request=lora_request,
+ use_tqdm=False,
+ )
+ output_text = outputs[0].outputs[0].text
+ return output_text
+
+ def reload_lora(name: str):
+ """
+ reload a lora to simulate the case:
+ setting `VLLM_ALLOW_RUNTIME_LORA_UPDATING=true`
+ for dynamic lora loading and unloading
+ """
+ remove_lora_response = llm.llm_engine.remove_lora(
+ lora_id=LORA_NAME_ID_MAP[name]
+ )
+
+ add_lora_response = llm.llm_engine.add_lora(
+ make_add_lora_request(name, LORA_NAME_PATH_MAP[name])
+ )
+
+ print(f"{remove_lora_response=}, {add_lora_response=}")
+
+ def check_outputs(outputs: str, expected: str):
+ print(f"{prompt=}.\n{expected_output=}\n{output_text=}")
+ print("\n----------------------------\n")
+ assert outputs == expected
+
+ for prompt, expected_output in zip(LORA_TEST_PROMPTS, LORA_TEST_EXPECTED):
+ output_text = call_llm_get_outputs(prompt, "Alice")
+ check_outputs(output_text, expected_output)
+
+ # call Bob, ignore what it is output
+ call_llm_get_outputs(prompt, "Bob")
+ print("After call Bob:")
+
+ # call Alice
+ output_text = call_llm_get_outputs(prompt, "Alice")
+ check_outputs(output_text, expected_output)
+
+ # reload Bob Lora
+ reload_lora("Bob")
+ print("After reload Bob:")
+
+ # call Alice
+ output_text = call_llm_get_outputs(prompt, "Alice")
+ check_outputs(output_text, expected_output)
+
+ # reload Alice Lora
+ reload_lora("Alice")
+ print("After reload Alice:")
+
+ output_text = call_llm_get_outputs(prompt, "Alice")
+ check_outputs(output_text, expected_output)
+
+
+def test_multiple_lora_requests():
+ llm = LLM(
+ model=MODEL_PATH,
+ enable_lora=True,
+ max_loras=4,
+ max_lora_rank=LORA_RANK,
+ max_model_len=512,
+ gpu_memory_utilization=0.5,
+ )
+ PROMPTS = ["Hello, my name is"] * 2
+ LORA_NAME = "Alice"
+ lora_request = [
+ LoRARequest(LORA_NAME + str(idx), idx + 1, LORA_NAME_PATH_MAP[LORA_NAME])
+ for idx in range(len(PROMPTS))
+ ]
+ # Multiple SamplingParams should be matched with each prompt
+ outputs = llm.generate(PROMPTS, lora_request=lora_request)
+ assert len(PROMPTS) == len(outputs)
+
+ # Exception raised, if the size of params does not match the size of prompts
+ with pytest.raises(ValueError):
+ outputs = llm.generate(PROMPTS, lora_request=lora_request[:1])
+
+ # Single LoRARequest should be applied to every prompt
+ single_lora_request = lora_request[0]
+ outputs = llm.generate(PROMPTS, lora_request=single_lora_request)
+ assert len(PROMPTS) == len(outputs)
+
+
+def test_load_inplace_offline_reload(
+ qwen3_meowing_lora_files: str, qwen3_woofing_lora_files: str
+) -> None:
+ """
+ Test that load_inplace=True allows reloading LoRA adapters with the same ID
+ in offline mode (using LLM class directly).
+ """
+ llm = LLM(
+ model=MODEL_PATH,
+ enable_lora=True,
+ max_loras=2,
+ max_lora_rank=LORA_RANK,
+ max_model_len=512,
+ gpu_memory_utilization=0.5,
+ )
+ adapter_id = 1
+ messages = format_chatml_messages(
+ "Make your favorite animal noise.",
+ system_prompt="Follow the instructions to make animal noises",
+ )
+ sampling_params = SamplingParams(temperature=0, max_tokens=10)
+
+ # Load meowing LoRA with load_inplace=True
+ meowing_request = LoRARequest(
+ lora_name="test-adapter",
+ lora_int_id=adapter_id,
+ lora_path=qwen3_meowing_lora_files,
+ )
+
+ outputs = llm.chat([messages], sampling_params, lora_request=meowing_request)
+ first_output = outputs[0].outputs[0].text.strip()
+ assert "Meow Meow Meow" in first_output, (
+ f"Expected meowing output, got: {first_output}"
+ )
+
+ # Reload with woofing LoRA (same ID, different weights, load_inplace=True)
+ woofing_request = LoRARequest(
+ lora_name="test-adapter-woof",
+ lora_int_id=adapter_id, # Same ID
+ lora_path=qwen3_woofing_lora_files, # Different weights
+ load_inplace=True, # Force reload
+ )
+
+ outputs = llm.chat([messages], sampling_params, lora_request=woofing_request)
+ second_output = outputs[0].outputs[0].text.strip()
+ assert "Woof Woof Woof" in second_output, (
+ f"Expected woofing output, got: {second_output}"
+ )
+
+
+def test_load_inplace_false_no_reload(
+ qwen3_meowing_lora_files: str, qwen3_woofing_lora_files: str
+) -> None:
+ """
+ Test that load_inplace=False prevents reloading when an adapter
+ with the same ID already exists.
+ """
+ llm = LLM(
+ model=MODEL_PATH,
+ enable_lora=True,
+ max_loras=2,
+ max_lora_rank=LORA_RANK,
+ max_model_len=512,
+ gpu_memory_utilization=0.5,
+ )
+ adapter_id = 2
+ messages = format_chatml_messages(
+ "Make your favorite animal noise.",
+ system_prompt="Follow the instructions to make animal noises",
+ )
+ sampling_params = SamplingParams(temperature=0, max_tokens=10)
+
+ # Load meowing LoRA first with load_inplace=True
+ meowing_request_initial = LoRARequest(
+ lora_name="test-adapter-2",
+ lora_int_id=adapter_id,
+ lora_path=qwen3_meowing_lora_files,
+ )
+
+ outputs = llm.chat(
+ [messages], sampling_params, lora_request=meowing_request_initial
+ )
+ first_output = outputs[0].outputs[0].text.strip()
+ assert "Meow Meow Meow" in first_output, (
+ f"Expected meowing output, got: {first_output}"
+ )
+
+ # Try to load woofing LoRA with same ID but load_inplace=False
+ # This should NOT reload (adapter 2 already exists)
+ woofing_request_no_reload = LoRARequest(
+ lora_name="test-adapter-2-woof",
+ lora_int_id=adapter_id, # Same ID
+ lora_path=qwen3_woofing_lora_files,
+ )
+
+ outputs = llm.chat(
+ [messages], sampling_params, lora_request=woofing_request_no_reload
+ )
+ second_output = outputs[0].outputs[0].text.strip()
+ # Should still get meowing output because it didn't reload
+ assert "Meow Meow Meow" in second_output, (
+ f"Expected meowing output (no reload), got: {second_output}"
+ )
diff --git a/asian_ai/performance/vLLM/tests/lora/test_qwen3moe_tp.py b/asian_ai/performance/vLLM/tests/lora/test_qwen3moe_tp.py
new file mode 100644
index 0000000000000000000000000000000000000000..f736c343c8a8d07483c83e56aab109271f39abeb
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_qwen3moe_tp.py
@@ -0,0 +1,122 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+
+# NOTE To avoid overloading the CI pipeline, this test script will not
+# be triggered on CI and is primarily intended for local testing and verification.
+
+import pytest
+
+import vllm
+from vllm.lora.request import LoRARequest
+
+from ..utils import multi_gpu_test
+
+MODEL_PATH = "Qwen/Qwen3-30B-A3B"
+
+PROMPT_TEMPLATE = """<|im_start|>user
+I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.
+"
+##Instruction:
+candidate_poll contains tables such as candidate, people. Table candidate has columns such as Candidate_ID, People_ID, Poll_Source, Date, Support_rate, Consider_rate, Oppose_rate, Unsure_rate. Candidate_ID is the primary key.
+Table people has columns such as People_ID, Sex, Name, Date_of_Birth, Height, Weight. People_ID is the primary key.
+The People_ID of candidate is the foreign key of People_ID of people.
+
+
+###Input:
+{context}
+
+###Response:<|im_end|>
+<|im_start|>assistant""" # noqa: E501
+
+EXPECTED_LORA_OUTPUT = [
+ "\n\n\n\nSELECT count(*) FROM candidate",
+ "\n\n\n\nSELECT count(*) FROM candidate",
+ "\n\n\n\nSELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1", # noqa: E501
+ "\n\n\n\nSELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1", # noqa: E501
+]
+
+
+def generate_and_test(llm: vllm.LLM, lora_path: str, lora_id: int) -> None:
+ prompts = [
+ PROMPT_TEMPLATE.format(context="How many candidates are there?"),
+ PROMPT_TEMPLATE.format(context="Count the number of candidates."),
+ PROMPT_TEMPLATE.format(
+ context="Which poll resource provided the most number of candidate information?" # noqa: E501
+ ),
+ PROMPT_TEMPLATE.format(
+ context="Return the poll resource associated with the most candidates."
+ ),
+ ]
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=64)
+ outputs = llm.generate(
+ prompts,
+ sampling_params,
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
+ )
+ # Print the outputs.
+ generated_texts: list[str] = []
+ for output in outputs:
+ prompt = output.prompt
+ generated_text = output.outputs[0].text.strip()
+ generated_texts.append(generated_text)
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
+ assert generated_texts[i].startswith(EXPECTED_LORA_OUTPUT[i])
+
+
+def test_qwen3moe_lora(qwen3moe_lora_files):
+ # We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
+ # Otherwise, the lora-test will fail due to CUDA OOM.
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enable_lora=True,
+ max_loras=4,
+ enforce_eager=True,
+ trust_remote_code=True,
+ enable_chunked_prefill=True,
+ enable_mixed_moe_lora_format=True,
+ )
+
+ generate_and_test(llm, qwen3moe_lora_files, lora_id=1)
+ generate_and_test(llm, qwen3moe_lora_files, lora_id=2)
+
+
+@multi_gpu_test(num_gpus=2)
+@pytest.mark.parametrize("ep", [False, True])
+@pytest.mark.parametrize("enable_mixed_moe_lora_format", [False, True])
+def test_qwen3moe_lora_tp2(ep, qwen3moe_lora_files, enable_mixed_moe_lora_format):
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enable_lora=True,
+ max_loras=4,
+ trust_remote_code=True,
+ enable_expert_parallel=ep,
+ tensor_parallel_size=2,
+ enable_mixed_moe_lora_format=enable_mixed_moe_lora_format,
+ )
+
+ generate_and_test(llm, qwen3moe_lora_files, lora_id=1)
+ generate_and_test(llm, qwen3moe_lora_files, lora_id=2)
+
+
+@multi_gpu_test(num_gpus=4)
+@pytest.mark.parametrize("ep", [False, True])
+@pytest.mark.parametrize("enable_mixed_moe_lora_format", [False, True])
+def test_qwen3moe_lora_tp4(ep, qwen3moe_lora_files, enable_mixed_moe_lora_format):
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enable_lora=True,
+ max_loras=4,
+ trust_remote_code=True,
+ enable_expert_parallel=ep,
+ tensor_parallel_size=4,
+ enable_mixed_moe_lora_format=enable_mixed_moe_lora_format,
+ )
+
+ generate_and_test(llm, qwen3moe_lora_files, lora_id=1)
+ generate_and_test(llm, qwen3moe_lora_files, lora_id=2)
diff --git a/asian_ai/performance/vLLM/tests/lora/test_qwenvl.py b/asian_ai/performance/vLLM/tests/lora/test_qwenvl.py
new file mode 100644
index 0000000000000000000000000000000000000000..5432e0f82feb2be44acb2cd39ec212ce96d49e6c
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_qwenvl.py
@@ -0,0 +1,320 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from dataclasses import dataclass
+
+import pytest
+from packaging.version import Version
+from transformers import __version__ as TRANSFORMERS_VERSION
+
+import vllm
+from vllm.assets.image import ImageAsset
+from vllm.lora.request import LoRARequest
+from vllm.platforms import current_platform
+from vllm.sampling_params import BeamSearchParams
+
+
+@dataclass
+class TestConfig:
+ model_path: str
+ lora_path: str
+ max_num_seqs: int = 2
+ max_loras: int = 2
+ max_lora_rank: int = 32
+ enable_tower_connector_lora: bool = False
+ max_model_len: int = 8192
+ gpu_memory_utilization: float = 0.85
+ mm_processor_kwargs: dict[str, object] | None = None
+ mm_processor_cache_gb: float = 4
+
+ def __post_init__(self):
+ if self.mm_processor_kwargs is None:
+ # There is a bug in transformers v4 where size is ignored by
+ # `Qwen2VLProcessor.__call__`
+ if Version(TRANSFORMERS_VERSION) < Version("5.2.0"):
+ self.mm_processor_kwargs = {
+ "min_pixels": 28 * 28,
+ "max_pixels": 1280 * 28 * 28,
+ }
+ else:
+ self.mm_processor_kwargs = {
+ "size": {
+ "shortest_edge": 28 * 28,
+ "longest_edge": 1280 * 28 * 28,
+ }
+ }
+
+
+class Qwen2VLTester:
+ """Test helper for Qwen2 VL models with LoRA"""
+
+ PROMPT_TEMPLATE = (
+ "<|im_start|>system\nYou are a helpful assistant.<|im_end|>"
+ "\n<|im_start|>user\n<|vision_start|><|image_pad|><|vision_end|>"
+ "What is in the image?<|im_end|>\n"
+ "<|im_start|>assistant\n"
+ )
+
+ def __init__(self, config: TestConfig):
+ self.config = config
+ self.llm = self._initialize_llm()
+
+ def _initialize_llm(self) -> vllm.LLM:
+ """Initialize the LLM with given configuration"""
+ return vllm.LLM(
+ model=self.config.model_path,
+ max_num_seqs=self.config.max_num_seqs,
+ enable_lora=True,
+ max_loras=self.config.max_loras,
+ max_lora_rank=self.config.max_lora_rank,
+ enable_tower_connector_lora=self.config.enable_tower_connector_lora,
+ trust_remote_code=True,
+ gpu_memory_utilization=self.config.gpu_memory_utilization,
+ mm_processor_kwargs=self.config.mm_processor_kwargs,
+ mm_processor_cache_gb=self.config.mm_processor_cache_gb,
+ max_model_len=self.config.max_model_len,
+ )
+
+ def run_test(
+ self,
+ images: list[ImageAsset],
+ expected_outputs: list[str],
+ lora_id: int | None = None,
+ lora_name: str | None = None,
+ temperature: float = 0,
+ max_tokens: int = 5,
+ ):
+ sampling_params = vllm.SamplingParams(
+ temperature=temperature,
+ max_tokens=max_tokens,
+ )
+ inputs = [
+ {
+ "prompt": self.PROMPT_TEMPLATE,
+ "multi_modal_data": {"image": asset.pil_image},
+ }
+ for asset in images
+ ]
+
+ lora_request = LoRARequest(
+ lora_name if lora_name else str(lora_id), lora_id, self.config.lora_path
+ )
+ outputs = self.llm.generate(inputs, sampling_params, lora_request=lora_request)
+ generated_texts = [output.outputs[0].text.strip() for output in outputs]
+ # Validate outputs
+ for generated, expected in zip(generated_texts, expected_outputs):
+ assert expected.startswith(generated), (
+ f"Generated text {generated} doesn't match expected pattern {expected}"
+ )
+
+ def run_beam_search_test(
+ self,
+ images: list[ImageAsset],
+ expected_outputs: list[list[str]],
+ lora_id: int | None = None,
+ temperature: float = 0,
+ beam_width: int = 2,
+ max_tokens: int = 5,
+ ):
+ beam_search_params = BeamSearchParams(
+ beam_width=beam_width, max_tokens=max_tokens, temperature=temperature
+ )
+
+ inputs = [
+ {
+ "prompt": self.PROMPT_TEMPLATE,
+ "multi_modal_data": {"image": asset.pil_image},
+ }
+ for asset in images
+ ]
+
+ lora_request = LoRARequest(str(lora_id), lora_id, self.config.lora_path)
+ outputs = self.llm.beam_search(
+ inputs, beam_search_params, lora_request=lora_request
+ )
+
+ for output_obj, expected_texts in zip(outputs, expected_outputs):
+ output_texts = [seq.text for seq in output_obj.sequences]
+
+ for output_text, expected_text in zip(output_texts, expected_texts):
+ # NOTE beam search .text contains the whole text including inputs
+ assert output_text.endswith(expected_text), (
+ f"Generated {output_text} does not match expected {expected_text}"
+ )
+
+
+TEST_IMAGES = [
+ ImageAsset("stop_sign"),
+ ImageAsset("cherry_blossom"),
+]
+
+EXPECTED_OUTPUTS = [
+ "A red stop sign stands prominently in the foreground, with a traditional Chinese gate and a black SUV in the background, illustrating a blend of modern and cultural elements.", # noqa: E501
+ "A majestic skyscraper stands tall, partially obscured by a vibrant canopy of cherry blossoms, against a clear blue sky.", # noqa: E501
+]
+
+EXPECTED_OUTPUTS_LANGUAGE = [
+ "A stop sign is shown in an Asian city, with buildings and a car in the "
+ "background.",
+ "The Tokyo Skytree can be seen behind the pink blossoms of the cherry trees.",
+]
+
+EXPECTED_OUTPUTS_VISION = [
+ "A stop sign in front of oriental buildings.",
+ "A tree with pink flowers in front of it and a blue sky behind the flowers.",
+]
+
+EXPECTED_OUTPUTS_VISION_NO_CONNECTOR = [
+ "A stop sign is located on the street of a Chinese neighborhood.",
+ "A closeup shot of the Tokyo Skytree with pink flowers in the foreground.",
+]
+
+EXPECTED_BEAM_SEARCH_OUTPUTS = [
+ [
+ "A majestic skyscraper stands",
+ "A majestic tower stands tall",
+ ],
+]
+
+QWEN2VL_MODEL_PATH = "Qwen/Qwen2-VL-2B-Instruct"
+QWEN25VL_MODEL_PATH = "Qwen/Qwen2.5-VL-3B-Instruct"
+QWEN3VL_MODEL_PATH = "Qwen/Qwen3-VL-4B-Instruct"
+
+
+def test_qwen2vl_lora(qwen2vl_lora_files):
+ """Test Qwen 2.0 VL model with LoRA"""
+ config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files)
+ tester = Qwen2VLTester(config)
+
+ # Test with different LoRA IDs
+ for lora_id in [1, 2]:
+ tester.run_test(TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id)
+
+
+def test_qwen2vl_lora_beam_search(qwen2vl_lora_files):
+ """Test Qwen 2.0 VL model with LoRA through beam search."""
+ config = TestConfig(model_path=QWEN2VL_MODEL_PATH, lora_path=qwen2vl_lora_files)
+ tester = Qwen2VLTester(config)
+
+ # Test with different LoRA IDs
+ for lora_id in [1, 2]:
+ # NOTE currently, we only test cherry blossom since stop sign
+ # output is slightly different for v1; - the root cause is likely
+ # independent of the intent of this test, which is to ensure beam
+ # search passes through lora through correctly.
+ tester.run_beam_search_test(
+ [ImageAsset("cherry_blossom")],
+ expected_outputs=EXPECTED_BEAM_SEARCH_OUTPUTS,
+ lora_id=lora_id,
+ )
+
+
+@pytest.mark.skipif(
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
+)
+def test_qwen25vl_lora(qwen25vl_lora_files):
+ """Test Qwen 2.5 VL model with LoRA"""
+ config = TestConfig(model_path=QWEN25VL_MODEL_PATH, lora_path=qwen25vl_lora_files)
+ tester = Qwen2VLTester(config)
+
+ # Test with different LoRA IDs
+ for lora_id in [1, 2]:
+ tester.run_test(TEST_IMAGES, expected_outputs=EXPECTED_OUTPUTS, lora_id=lora_id)
+
+
+@pytest.mark.skipif(
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
+)
+def test_qwen25vl_vision_lora(qwen25vl_vision_lora_files):
+ config = TestConfig(
+ model_path=QWEN25VL_MODEL_PATH,
+ lora_path=qwen25vl_vision_lora_files,
+ # Currently, tower_connector_lora is incompatible with
+ # the multi-modal processor cache.
+ # TODO: Remove this restriction
+ mm_processor_cache_gb=0,
+ enable_tower_connector_lora=True,
+ )
+ tester = Qwen2VLTester(config)
+ for lora_id in [1, 2]:
+ tester.run_test(
+ TEST_IMAGES,
+ expected_outputs=EXPECTED_OUTPUTS,
+ lora_id=lora_id,
+ )
+
+
+def test_qwen3vl_vision_lora(qwen3vl_vision_lora_files):
+ config = TestConfig(
+ model_path=QWEN3VL_MODEL_PATH,
+ lora_path=qwen3vl_vision_lora_files,
+ # Currently, tower_connector_lora is incompatible with
+ # the multi-modal processor cache.
+ # TODO: Remove this restriction
+ mm_processor_cache_gb=0,
+ enable_tower_connector_lora=True,
+ )
+ tester = Qwen2VLTester(config)
+ for lora_id in [1, 2]:
+ tester.run_test(
+ TEST_IMAGES,
+ expected_outputs=EXPECTED_OUTPUTS,
+ lora_id=lora_id,
+ )
+
+
+def test_qwen2vl_multiple_lora_types(
+ qwen2vl_language_lora_files,
+ qwen2vl_vision_tower_connector_lora_files,
+ qwen2vl_vision_tower_lora_files,
+):
+ """
+ Test multiple LoRA adapter types (language, vision tower + connector,
+ vision tower only) using the same LLM instance to verify mm_encoder_cache
+ behavior with different LoRA requests.
+
+ By reusing the same LLM instance across different LoRA requests, we ensure that
+ the multimodal encoder cache correctly manages state transitions between
+ language-only and vision-enabled LoRA adapters.
+ """
+ config = TestConfig(
+ model_path=QWEN2VL_MODEL_PATH,
+ # We'll override the lora_path for each specific test, but need to provide
+ # an initial path for initialization
+ lora_path=qwen2vl_language_lora_files,
+ # Currently, tower_connector_lora is incompatible with
+ # the multi-modal processor cache.
+ # TODO: Remove this restriction
+ mm_processor_cache_gb=0,
+ enable_tower_connector_lora=True,
+ )
+ tester = Qwen2VLTester(config)
+
+ # Test 1: Language-only LoRA adapter
+ tester.config.lora_path = qwen2vl_language_lora_files
+ for lora_id in [1, 2]:
+ tester.run_test(
+ TEST_IMAGES,
+ expected_outputs=EXPECTED_OUTPUTS_LANGUAGE,
+ lora_id=lora_id,
+ lora_name="language_only",
+ )
+
+ # Test 2: Vision tower + connector LoRA adapter
+ tester.config.lora_path = qwen2vl_vision_tower_connector_lora_files
+ for lora_id in [3, 4]:
+ tester.run_test(
+ TEST_IMAGES,
+ expected_outputs=EXPECTED_OUTPUTS_VISION,
+ lora_id=lora_id,
+ lora_name="vision_tower_connector",
+ )
+
+ # Test 3: Vision tower only LoRA adapter (no connector)
+ tester.config.lora_path = qwen2vl_vision_tower_lora_files
+ for lora_id in [5, 6]:
+ tester.run_test(
+ TEST_IMAGES,
+ expected_outputs=EXPECTED_OUTPUTS_VISION_NO_CONNECTOR,
+ lora_id=lora_id,
+ lora_name="vision_tower",
+ )
diff --git a/asian_ai/performance/vLLM/tests/lora/test_resolver.py b/asian_ai/performance/vLLM/tests/lora/test_resolver.py
new file mode 100644
index 0000000000000000000000000000000000000000..08eb4590496b1a876e444b70adbfbf6cea231bb7
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_resolver.py
@@ -0,0 +1,75 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+
+import pytest
+
+from vllm.lora.request import LoRARequest
+from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
+
+
+class DummyLoRAResolver(LoRAResolver):
+ """A dummy LoRA resolver for testing."""
+
+ async def resolve_lora(
+ self, base_model_name: str, lora_name: str
+ ) -> LoRARequest | None:
+ if lora_name == "test_lora":
+ return LoRARequest(
+ lora_name=lora_name,
+ lora_path=f"/dummy/path/{base_model_name}/{lora_name}",
+ lora_int_id=abs(hash(lora_name)),
+ )
+ return None
+
+
+def test_resolver_registry_registration():
+ """Test basic resolver registration functionality."""
+ registry = LoRAResolverRegistry
+ resolver = DummyLoRAResolver()
+
+ # Register a new resolver
+ registry.register_resolver("dummy", resolver)
+ assert "dummy" in registry.get_supported_resolvers()
+
+ # Get registered resolver
+ retrieved_resolver = registry.get_resolver("dummy")
+ assert retrieved_resolver is resolver
+
+
+def test_resolver_registry_duplicate_registration():
+ """Test registering a resolver with an existing name."""
+ registry = LoRAResolverRegistry
+ resolver1 = DummyLoRAResolver()
+ resolver2 = DummyLoRAResolver()
+
+ registry.register_resolver("dummy", resolver1)
+ registry.register_resolver("dummy", resolver2)
+
+ assert registry.get_resolver("dummy") is resolver2
+
+
+def test_resolver_registry_unknown_resolver():
+ """Test getting a non-existent resolver."""
+ registry = LoRAResolverRegistry
+
+ with pytest.raises(KeyError, match="not found"):
+ registry.get_resolver("unknown_resolver")
+
+
+@pytest.mark.asyncio
+async def test_dummy_resolver_resolve():
+ """Test the dummy resolver's resolve functionality."""
+ dummy_resolver = DummyLoRAResolver()
+ base_model_name = "base_model_test"
+ lora_name = "test_lora"
+
+ # Test successful resolution
+ result = await dummy_resolver.resolve_lora(base_model_name, lora_name)
+ assert isinstance(result, LoRARequest)
+ assert result.lora_name == lora_name
+ assert result.lora_path == f"/dummy/path/{base_model_name}/{lora_name}"
+
+ # Test failed resolution
+ result = await dummy_resolver.resolve_lora(base_model_name, "nonexistent_lora")
+ assert result is None
diff --git a/asian_ai/performance/vLLM/tests/lora/test_transformers_model.py b/asian_ai/performance/vLLM/tests/lora/test_transformers_model.py
new file mode 100644
index 0000000000000000000000000000000000000000..2b79c7a511ab473d8be0209c341411a1c173a6eb
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_transformers_model.py
@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+import vllm
+from vllm.lora.request import LoRARequest
+from vllm.platforms import current_platform
+
+from ..utils import create_new_process_for_each_test, multi_gpu_test
+
+MODEL_PATH = "hmellor/Ilama-3.2-1B"
+
+PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:""" # noqa: E501
+
+EXPECTED_LORA_OUTPUT = [
+ "SELECT count(*) FROM singer",
+ "SELECT avg(age) , min(age) , max(age) FROM singer WHERE country = 'France'", # noqa: E501
+ "SELECT DISTINCT Country FROM singer WHERE Age > 20",
+]
+
+
+def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
+ prompts = [
+ PROMPT_TEMPLATE.format(query="How many singers do we have?"),
+ PROMPT_TEMPLATE.format(
+ query="What is the average, minimum, and maximum age of all singers from France?" # noqa: E501
+ ),
+ PROMPT_TEMPLATE.format(
+ query="What are all distinct countries where singers above age 20 are from?" # noqa: E501
+ ),
+ ]
+ sampling_params = vllm.SamplingParams(temperature=0, max_tokens=32)
+ outputs = llm.generate(
+ prompts,
+ sampling_params,
+ lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
+ )
+ # Print the outputs.
+ generated_texts: list[str] = []
+ for output in outputs:
+ prompt = output.prompt
+ generated_text = output.outputs[0].text.strip()
+ generated_texts.append(generated_text)
+ print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+ return generated_texts
+
+
+def test_ilama_lora(ilama_lora_files):
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enable_lora=True,
+ max_loras=4,
+ max_lora_rank=16,
+ trust_remote_code=True,
+ enable_chunked_prefill=True,
+ )
+
+ output1 = do_sample(llm, ilama_lora_files, lora_id=1)
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
+ assert output1[i] == EXPECTED_LORA_OUTPUT[i]
+ output2 = do_sample(llm, ilama_lora_files, lora_id=2)
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
+ assert output2[i] == EXPECTED_LORA_OUTPUT[i]
+
+
+@pytest.mark.skipif(
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
+)
+@multi_gpu_test(num_gpus=4)
+@create_new_process_for_each_test()
+def test_ilama_lora_tp4(ilama_lora_files):
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enable_lora=True,
+ max_loras=4,
+ max_lora_rank=16,
+ tensor_parallel_size=4,
+ trust_remote_code=True,
+ fully_sharded_loras=False,
+ enable_chunked_prefill=True,
+ )
+
+ output1 = do_sample(llm, ilama_lora_files, lora_id=1)
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
+ assert output1[i] == EXPECTED_LORA_OUTPUT[i]
+ output2 = do_sample(llm, ilama_lora_files, lora_id=2)
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
+ assert output2[i] == EXPECTED_LORA_OUTPUT[i]
+
+
+@pytest.mark.skipif(
+ current_platform.is_cuda_alike(), reason="Skipping to avoid redundant model tests"
+)
+@multi_gpu_test(num_gpus=4)
+@create_new_process_for_each_test()
+def test_ilama_lora_tp4_fully_sharded_loras(ilama_lora_files):
+ llm = vllm.LLM(
+ MODEL_PATH,
+ max_model_len=1024,
+ enable_lora=True,
+ max_loras=4,
+ max_lora_rank=16,
+ tensor_parallel_size=4,
+ trust_remote_code=True,
+ fully_sharded_loras=True,
+ enable_chunked_prefill=True,
+ )
+ output1 = do_sample(llm, ilama_lora_files, lora_id=1)
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
+ assert output1[i] == EXPECTED_LORA_OUTPUT[i]
+ output2 = do_sample(llm, ilama_lora_files, lora_id=2)
+ for i in range(len(EXPECTED_LORA_OUTPUT)):
+ assert output2[i] == EXPECTED_LORA_OUTPUT[i]
diff --git a/asian_ai/performance/vLLM/tests/lora/test_utils.py b/asian_ai/performance/vLLM/tests/lora/test_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..4cdb9651c734c9c13f2cccf7a0d172e6e1bd15b5
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_utils.py
@@ -0,0 +1,201 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from collections import OrderedDict
+from typing import NamedTuple
+from unittest.mock import MagicMock, patch
+
+import pytest
+from huggingface_hub.utils import HfHubHTTPError
+from torch import nn
+
+from vllm.lora.utils import (
+ get_adapter_absolute_path,
+ parse_fine_tuned_lora_name,
+ replace_submodule,
+)
+from vllm.model_executor.models.utils import WeightsMapper
+
+
+class LoRANameParserTestConfig(NamedTuple):
+ name: str
+ module_name: str
+ is_lora_a: bool
+ weights_mapper: WeightsMapper | None = None
+
+
+def test_parse_fine_tuned_lora_name_valid():
+ fixture = [
+ LoRANameParserTestConfig(
+ "base_model.model.lm_head.lora_A.weight", "lm_head", True, False
+ ),
+ LoRANameParserTestConfig(
+ "base_model.model.lm_head.lora_B.weight", "lm_head", False, False
+ ),
+ LoRANameParserTestConfig(
+ "base_model.model.model.embed_tokens.lora_embedding_A",
+ "model.embed_tokens",
+ True,
+ ),
+ LoRANameParserTestConfig(
+ "base_model.model.model.embed_tokens.lora_embedding_B",
+ "model.embed_tokens",
+ False,
+ ),
+ LoRANameParserTestConfig(
+ "base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
+ "model.layers.9.mlp.down_proj",
+ True,
+ ),
+ LoRANameParserTestConfig(
+ "base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
+ "model.layers.9.mlp.down_proj",
+ False,
+ ),
+ LoRANameParserTestConfig(
+ "language_model.layers.9.mlp.down_proj.lora_A.weight",
+ "language_model.layers.9.mlp.down_proj",
+ True,
+ ),
+ LoRANameParserTestConfig(
+ "language_model.layers.9.mlp.down_proj.lora_B.weight",
+ "language_model.layers.9.mlp.down_proj",
+ False,
+ ),
+ # Test with WeightsMapper
+ LoRANameParserTestConfig(
+ "base_model.model.model.layers.9.mlp.down_proj.lora_A.weight",
+ "language_model.model.layers.9.mlp.down_proj",
+ True,
+ weights_mapper=WeightsMapper(
+ orig_to_new_prefix={"model.": "language_model.model."}
+ ),
+ ),
+ LoRANameParserTestConfig(
+ "base_model.model.model.layers.9.mlp.down_proj.lora_B.weight",
+ "language_model.model.layers.9.mlp.down_proj",
+ False,
+ weights_mapper=WeightsMapper(
+ orig_to_new_prefix={"model.": "language_model.model."}
+ ),
+ ),
+ LoRANameParserTestConfig(
+ "model.layers.9.mlp.down_proj.lora_A.weight",
+ "language_model.model.layers.9.mlp.down_proj",
+ True,
+ weights_mapper=WeightsMapper(
+ orig_to_new_prefix={"model.": "language_model.model."}
+ ),
+ ),
+ LoRANameParserTestConfig(
+ "model.layers.9.mlp.down_proj.lora_B.weight",
+ "language_model.model.layers.9.mlp.down_proj",
+ False,
+ weights_mapper=WeightsMapper(
+ orig_to_new_prefix={"model.": "language_model.model."}
+ ),
+ ),
+ ]
+ for name, module_name, is_lora_a, weights_mapper in fixture:
+ assert (module_name, is_lora_a) == parse_fine_tuned_lora_name(
+ name, weights_mapper
+ )
+
+
+def test_parse_fine_tuned_lora_name_invalid():
+ fixture = {
+ "base_model.weight",
+ "base_model.model.weight",
+ }
+ for name in fixture:
+ with pytest.raises(ValueError, match="unsupported LoRA weight"):
+ parse_fine_tuned_lora_name(name)
+
+
+def test_replace_submodule():
+ model = nn.Sequential(
+ OrderedDict(
+ [
+ ("dense1", nn.Linear(764, 100)),
+ ("act1", nn.ReLU()),
+ ("dense2", nn.Linear(100, 50)),
+ (
+ "seq1",
+ nn.Sequential(
+ OrderedDict(
+ [
+ ("dense1", nn.Linear(100, 10)),
+ ("dense2", nn.Linear(10, 50)),
+ ]
+ )
+ ),
+ ),
+ ("act2", nn.ReLU()),
+ ("output", nn.Linear(50, 10)),
+ ("outact", nn.Sigmoid()),
+ ]
+ )
+ )
+
+ sigmoid = nn.Sigmoid()
+
+ replace_submodule(model, "act1", sigmoid)
+ assert dict(model.named_modules())["act1"] == sigmoid
+
+ dense2 = nn.Linear(1, 5)
+ replace_submodule(model, "seq1.dense2", dense2)
+ assert dict(model.named_modules())["seq1.dense2"] == dense2
+
+
+# Unit tests for get_adapter_absolute_path
+@patch("os.path.isabs")
+def test_get_adapter_absolute_path_absolute(mock_isabs):
+ path = "/absolute/path/to/lora"
+ mock_isabs.return_value = True
+ assert get_adapter_absolute_path(path) == path
+
+
+@patch("os.path.expanduser")
+def test_get_adapter_absolute_path_expanduser(mock_expanduser):
+ # Path with ~ that needs to be expanded
+ path = "~/relative/path/to/lora"
+ absolute_path = "/home/user/relative/path/to/lora"
+ mock_expanduser.return_value = absolute_path
+ assert get_adapter_absolute_path(path) == absolute_path
+
+
+@patch("os.path.exists")
+@patch("os.path.abspath")
+def test_get_adapter_absolute_path_local_existing(mock_abspath, mock_exist):
+ # Relative path that exists locally
+ path = "relative/path/to/lora"
+ absolute_path = "/absolute/path/to/lora"
+ mock_exist.return_value = True
+ mock_abspath.return_value = absolute_path
+ assert get_adapter_absolute_path(path) == absolute_path
+
+
+@patch("huggingface_hub.HfApi.snapshot_download")
+@patch("os.path.exists")
+def test_get_adapter_absolute_path_huggingface(mock_exist, mock_snapshot_download):
+ # Hugging Face model identifier
+ path = "org/repo"
+ absolute_path = "/mock/snapshot/path"
+ mock_exist.return_value = False
+ mock_snapshot_download.return_value = absolute_path
+ assert get_adapter_absolute_path(path) == absolute_path
+
+
+@patch("huggingface_hub.HfApi.snapshot_download")
+@patch("os.path.exists")
+def test_get_adapter_absolute_path_huggingface_error(
+ mock_exist, mock_snapshot_download
+):
+ # Hugging Face model identifier with download error
+ path = "org/repo"
+ mock_exist.return_value = False
+ mock_snapshot_download.side_effect = HfHubHTTPError(
+ "failed to query model info",
+ response=MagicMock(),
+ )
+ assert get_adapter_absolute_path(path) == path
diff --git a/asian_ai/performance/vLLM/tests/lora/test_whisper.py b/asian_ai/performance/vLLM/tests/lora/test_whisper.py
new file mode 100644
index 0000000000000000000000000000000000000000..444163405b0cde842c4675b7d247e7518177a106
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_whisper.py
@@ -0,0 +1,134 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Integration tests for Whisper models with LoRA adapters.
+
+These tests verify that Whisper models can correctly load and use LoRA adapters
+for speech-to-text transcription tasks.
+"""
+
+import pytest
+
+import vllm
+from vllm.assets.audio import AudioAsset
+from vllm.lora.request import LoRARequest
+from vllm.platforms import current_platform
+
+from ..utils import create_new_process_for_each_test
+
+# Model configuration
+WHISPER_MODEL = "openai/whisper-small"
+
+# Test prompts for Whisper transcription
+WHISPER_PROMPT = "<|startoftranscript|><|en|><|transcribe|><|notimestamps|>"
+
+# Note: whisper_lora_files fixture is defined in conftest.py
+
+
+@pytest.fixture(autouse=True)
+def use_spawn_for_whisper(monkeypatch):
+ """Whisper has issues with forked workers, use spawn instead."""
+ monkeypatch.setenv("VLLM_WORKER_MULTIPROC_METHOD", "spawn")
+
+
+def create_whisper_llm(
+ enable_lora: bool = True, max_loras: int = 2, attn_backend: str | None = None
+):
+ """Create a Whisper LLM instance with optional LoRA support."""
+ return vllm.LLM(
+ model=WHISPER_MODEL,
+ enable_lora=enable_lora,
+ max_loras=max_loras if enable_lora else 1,
+ max_lora_rank=64,
+ max_model_len=448,
+ dtype="half",
+ enforce_eager=True, # For stability in tests
+ attention_config={"backend": attn_backend},
+ )
+
+
+def run_whisper_inference(
+ llm: vllm.LLM,
+ lora_path: str | None = None,
+ lora_id: int = 1,
+) -> list[str]:
+ """Run Whisper inference with optional LoRA adapter."""
+ # Load test audio
+ audio_asset = AudioAsset("mary_had_lamb")
+ audio_data = audio_asset.audio_and_sample_rate
+
+ inputs = [
+ {
+ "prompt": WHISPER_PROMPT,
+ "multi_modal_data": {"audio": audio_data},
+ }
+ ]
+
+ sampling_params = vllm.SamplingParams(
+ temperature=0,
+ max_tokens=200,
+ )
+
+ # Prepare LoRA request if adapter path is provided
+ lora_request = None
+ if lora_path:
+ lora_request = LoRARequest(
+ lora_name=f"whisper_lora_{lora_id}",
+ lora_int_id=lora_id,
+ lora_path=lora_path,
+ )
+
+ outputs = llm.generate(inputs, sampling_params, lora_request=lora_request)
+
+ return [output.outputs[0].text for output in outputs]
+
+
+@create_new_process_for_each_test()
+def test_whisper_lora_inference(whisper_lora_files):
+ """Test basic Whisper inference with a LoRA adapter.
+
+ This test verifies that:
+ 1. Whisper model can be loaded with LoRA support enabled
+ 2. A LoRA adapter can be applied during inference
+ 3. The model produces valid transcription output
+ """
+ llm = create_whisper_llm(enable_lora=True)
+
+ # Run inference with LoRA
+ outputs = run_whisper_inference(llm, lora_path=whisper_lora_files, lora_id=1)
+
+ # Verify we got a non-empty transcription
+ assert len(outputs) == 1
+ assert len(outputs[0]) > 0, "Expected non-empty transcription output"
+
+ # The output should contain some recognizable words from the audio
+ # (Mary had a little lamb)
+ print(f"Transcription output: {outputs[0]}")
+
+
+@create_new_process_for_each_test()
+def test_whisper_multi_lora(whisper_lora_files):
+ """Test Whisper with multiple LoRA adapter IDs.
+
+ This test verifies that the same LoRA adapter can be loaded with
+ different IDs and produce consistent results.
+ """
+ llm = create_whisper_llm(
+ enable_lora=True,
+ max_loras=4,
+ attn_backend="TRITON_ATTN" if current_platform.is_rocm() else None,
+ )
+
+ # Test with different LoRA IDs using the same adapter
+ outputs_lora1 = run_whisper_inference(llm, lora_path=whisper_lora_files, lora_id=1)
+ outputs_lora2 = run_whisper_inference(llm, lora_path=whisper_lora_files, lora_id=2)
+
+ # Both should produce valid outputs
+ assert len(outputs_lora1[0]) > 0
+ assert len(outputs_lora2[0]) > 0
+
+ # Same adapter with different IDs should produce same output
+ assert outputs_lora1 == outputs_lora2, (
+ f"Expected same outputs for same adapter with different IDs. "
+ f"Got: {outputs_lora1} vs {outputs_lora2}"
+ )
diff --git a/asian_ai/performance/vLLM/tests/lora/test_worker.py b/asian_ai/performance/vLLM/tests/lora/test_worker.py
new file mode 100644
index 0000000000000000000000000000000000000000..63735784fe98bda0d313cf291219a944f84e29f7
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/test_worker.py
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import os
+import random
+import tempfile
+from unittest.mock import patch
+
+from vllm.config import (
+ CacheConfig,
+ DeviceConfig,
+ ModelConfig,
+ ParallelConfig,
+ SchedulerConfig,
+ VllmConfig,
+ set_current_vllm_config,
+)
+from vllm.config.load import LoadConfig
+from vllm.config.lora import LoRAConfig
+from vllm.lora.model_manager import LoRAMapping
+from vllm.lora.request import LoRARequest
+from vllm.platforms import current_platform
+
+if current_platform.is_xpu():
+ from vllm.v1.worker.xpu_worker import XPUWorker as Worker
+else:
+ from vllm.v1.worker.gpu_worker import Worker
+
+MODEL_PATH = "Qwen/Qwen3-0.6B"
+NUM_LORAS = 16
+
+DEVICE_TYPE = current_platform.device_type
+
+
+@patch.dict(os.environ, {"RANK": "0"})
+def test_worker_apply_lora(qwen3_lora_files):
+ def set_active_loras(worker: Worker, lora_requests: list[LoRARequest]):
+ lora_mapping = LoRAMapping([], [])
+
+ worker.model_runner.lora_manager.set_active_adapters(
+ lora_requests, lora_mapping
+ )
+
+ model_config = ModelConfig(
+ MODEL_PATH,
+ seed=0,
+ dtype="float16",
+ max_model_len=127,
+ enforce_eager=True,
+ )
+
+ vllm_config = VllmConfig(
+ model_config=model_config,
+ load_config=LoadConfig(
+ download_dir=None,
+ load_format="dummy",
+ ),
+ parallel_config=ParallelConfig(
+ pipeline_parallel_size=1,
+ tensor_parallel_size=1,
+ data_parallel_size=1,
+ ),
+ scheduler_config=SchedulerConfig(
+ max_model_len=model_config.max_model_len,
+ is_encoder_decoder=model_config.is_encoder_decoder,
+ runner_type="generate",
+ max_num_batched_tokens=32,
+ max_num_seqs=32,
+ max_num_partial_prefills=32,
+ ),
+ device_config=DeviceConfig(DEVICE_TYPE),
+ cache_config=CacheConfig(
+ block_size=16,
+ cache_dtype="auto",
+ ),
+ lora_config=LoRAConfig(
+ max_lora_rank=8, max_cpu_loras=NUM_LORAS, max_loras=NUM_LORAS
+ ),
+ )
+ worker = Worker(
+ vllm_config=vllm_config,
+ local_rank=0,
+ rank=0,
+ distributed_init_method=f"file://{tempfile.mkstemp()[1]}",
+ )
+
+ with set_current_vllm_config(vllm_config):
+ worker.init_device()
+ worker.load_model()
+
+ set_active_loras(worker, [])
+ assert worker.list_loras() == set()
+
+ lora_requests = [
+ LoRARequest(str(i + 1), i + 1, qwen3_lora_files) for i in range(NUM_LORAS)
+ ]
+
+ set_active_loras(worker, lora_requests)
+ assert worker.list_loras() == {
+ lora_request.lora_int_id for lora_request in lora_requests
+ }
+
+ for i in range(NUM_LORAS):
+ random.seed(i)
+ iter_lora_requests = random.choices(
+ lora_requests, k=random.randint(1, NUM_LORAS)
+ )
+ random.shuffle(iter_lora_requests)
+ iter_lora_requests = iter_lora_requests[: -random.randint(0, NUM_LORAS)]
+ set_active_loras(worker, lora_requests)
+ assert worker.list_loras().issuperset(
+ {lora_request.lora_int_id for lora_request in iter_lora_requests}
+ )
diff --git a/asian_ai/performance/vLLM/tests/lora/utils.py b/asian_ai/performance/vLLM/tests/lora/utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..84b0088554d8cf131089aac76bcbf2d8ebe2f328
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/lora/utils.py
@@ -0,0 +1,410 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import json
+import os
+from dataclasses import dataclass
+
+import torch
+from safetensors.torch import save_file
+
+from vllm.lora.lora_weights import LoRALayerWeights, PackedLoRALayerWeights
+from vllm.platforms import current_platform
+
+DEVICE_TYPE = current_platform.device_type
+
+
+class DummyLoRAManager:
+ def __init__(self, device: torch.device = f"{DEVICE_TYPE}:0"):
+ super().__init__()
+ self._loras: dict[str, LoRALayerWeights] = {}
+ self._device = device
+
+ def set_module_lora(self, module_name: str, lora: LoRALayerWeights):
+ self._loras[module_name] = lora
+
+ def get_module_lora(self, module_name: str) -> LoRALayerWeights:
+ return self._loras[module_name]
+
+ def init_random_lora(
+ self,
+ module_name: str,
+ weight: torch.Tensor,
+ rank: int = 8,
+ ):
+ lora = LoRALayerWeights(
+ module_name,
+ rank=rank,
+ lora_alpha=1,
+ lora_a=torch.rand(
+ [rank, weight.shape[1]], dtype=weight.dtype, device=self._device
+ ),
+ lora_b=torch.rand(
+ [weight.shape[0], rank], dtype=weight.dtype, device=self._device
+ ),
+ )
+ self.set_module_lora(module_name, lora)
+
+ return lora
+
+ def init_lora(
+ self,
+ module_name: str,
+ input_dim: int,
+ output_dim: int,
+ rank=8,
+ noop=False,
+ embeddings_tensor=None,
+ ):
+ lora = LoRALayerWeights(
+ module_name,
+ rank=rank,
+ lora_alpha=1,
+ lora_a=torch.rand([rank, input_dim], device=DEVICE_TYPE),
+ lora_b=torch.rand([output_dim, input_dim], device=DEVICE_TYPE),
+ embeddings_tensor=embeddings_tensor,
+ )
+ self.set_module_lora(module_name, lora)
+ return lora
+
+ def reset_lora(self):
+ self._loras = {}
+
+ def init_packed_lora(
+ self,
+ module_name: str,
+ input_dim: int,
+ output_dims: list[int],
+ noop_lora_index: list[int] | None = None,
+ rank: int = 8,
+ ):
+ base_loras: list[LoRALayerWeights] = []
+ noop_lora_index_set = set(noop_lora_index or [])
+
+ for i, out_dim in enumerate(output_dims):
+ base_lora = self.init_lora(
+ module_name + "_000_" + str(i),
+ input_dim,
+ out_dim,
+ rank=rank,
+ noop=i in noop_lora_index_set,
+ )
+ base_loras.append(base_lora)
+ packed_lora = PackedLoRALayerWeights.pack(base_loras)
+ self.set_module_lora(module_name, packed_lora)
+ return packed_lora
+
+
+def assert_close(a, b):
+ rtol, atol = {
+ torch.float16: (6e-2, 6e-2),
+ torch.bfloat16: (6e-2, 6e-2),
+ torch.float32: (1e-2, 1e-2),
+ }[a.dtype]
+ torch.testing.assert_close(a, b, rtol=rtol, atol=atol)
+
+
+@dataclass
+class PunicaTensors:
+ inputs_tensor: torch.Tensor
+ lora_weights: torch.Tensor | list[torch.Tensor]
+ our_out_tensor: torch.Tensor
+ ref_out_tensor: torch.Tensor
+ b_seq_start_loc: torch.Tensor
+ prompt_lora_mapping: torch.Tensor
+ seq_len_tensor: torch.Tensor
+ token_lora_mapping: torch.Tensor
+
+ def meta(self) -> tuple[int, int]:
+ """
+ Infer max_seq_length and token_nums from the tensors
+ and return them.
+ """
+ max_seq_length = self.seq_len_tensor.max()
+ token_nums = self.seq_len_tensor.sum().item()
+ if isinstance(max_seq_length, tuple):
+ max_seq_length = max_seq_length[0].item()
+ else:
+ max_seq_length = max_seq_length.item()
+ return max_seq_length, token_nums
+
+
+def generate_data(
+ batches,
+ hidden_size,
+ lora_nums,
+ max_rank,
+ seq_length,
+ dtype,
+ op_type,
+ device,
+) -> PunicaTensors:
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1, (batches,)).to(device)
+ b_seq_start_loc = torch.cumsum(
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
+ dim=0,
+ ).to(device)
+ total_tokens = seq_len_tensor.sum()
+ if op_type == "shrink":
+ inputs_tensor = torch.rand((total_tokens, hidden_size), dtype=dtype).to(device)
+ lora_weights = torch.rand(
+ (lora_nums, max_rank, hidden_size), # col-major
+ dtype=dtype,
+ ).to(device)
+ # shrink op need atomic_add, so output is initinized by 0
+ ref_out_tensor = torch.zeros(
+ (total_tokens, max_rank), dtype=dtype, device=inputs_tensor.device
+ )
+ # NOTE shrink kernel using torch.float32 as output type
+ our_out_tensor = torch.zeros((total_tokens, max_rank), dtype=torch.float32).to(
+ device
+ )
+ else:
+ inputs_tensor = torch.rand(
+ (total_tokens, max_rank),
+ dtype=dtype,
+ ).to(device)
+ lora_weights = torch.rand(
+ (lora_nums, hidden_size, max_rank), # col-major
+ dtype=dtype,
+ ).to(device)
+ # expand op needs to complete y+=a@lora_b, so output is
+ # initinized randomly
+ ref_out_tensor = torch.rand(
+ (total_tokens, hidden_size),
+ dtype=dtype,
+ ).to(device)
+ # Ensure the same input.
+ our_out_tensor = ref_out_tensor.clone()
+ lora_indices_tensor = torch.randint(
+ 0, lora_nums - 1 if lora_nums > 1 else 1, (batches,)
+ ).to(device)
+ indices = torch.zeros((total_tokens), dtype=torch.long).to(device)
+ current_offset = 0
+ for b_id in range(batches):
+ lora_index = lora_indices_tensor[b_id]
+ indices[current_offset : current_offset + seq_len_tensor[b_id]].copy_(
+ lora_index
+ )
+ current_offset += seq_len_tensor[b_id].item()
+
+ return PunicaTensors(
+ inputs_tensor,
+ lora_weights,
+ our_out_tensor,
+ ref_out_tensor,
+ b_seq_start_loc,
+ lora_indices_tensor,
+ seq_len_tensor,
+ indices,
+ )
+
+
+def generate_data_for_expand_nslices(
+ batches,
+ hidden_size,
+ lora_nums,
+ max_rank,
+ seq_length,
+ dtype,
+ nslices,
+ device,
+) -> PunicaTensors:
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1, (batches,)).to(device)
+ b_seq_start_loc = torch.cumsum(
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
+ dim=0,
+ ).to(device)
+ total_tokens = seq_len_tensor.sum()
+ inputs_tensor = torch.rand(
+ (total_tokens, max_rank),
+ dtype=dtype,
+ ).to(device)
+ lora_weights_lst = []
+ for _ in range(nslices):
+ lora_weights_lst.append(
+ torch.rand(
+ (lora_nums, hidden_size, max_rank), # col-major
+ dtype=dtype,
+ ).to(device)
+ )
+ # expand op needs to complete y+=a@lora_b, so output is
+ # initinized randomly
+ ref_out_tensor = torch.rand((total_tokens, hidden_size * nslices), dtype=dtype).to(
+ device
+ )
+ # Ensure the same input.
+ our_out_tensor = ref_out_tensor.clone()
+ lora_indices_tensor = torch.randint(
+ 0, lora_nums - 1 if lora_nums > 1 else 1, (batches,)
+ )
+ indices = torch.zeros((total_tokens), dtype=torch.long).to(device)
+ current_offset = 0
+ for b_id in range(batches):
+ lora_index = lora_indices_tensor[b_id]
+ indices[current_offset : current_offset + seq_len_tensor[b_id]] = (
+ lora_index.item()
+ )
+ current_offset += seq_len_tensor[b_id].item()
+
+ lora_indices_tensor = lora_indices_tensor.to(device)
+ return PunicaTensors(
+ inputs_tensor,
+ lora_weights_lst,
+ our_out_tensor,
+ ref_out_tensor,
+ b_seq_start_loc,
+ lora_indices_tensor,
+ seq_len_tensor,
+ indices,
+ )
+
+
+def generate_data_for_nslices(
+ batches,
+ hidden_size,
+ lora_nums,
+ max_rank,
+ seq_length,
+ nslices,
+ dtype,
+ op_type,
+ device,
+) -> PunicaTensors:
+ seq_len_tensor = torch.randint(seq_length, seq_length + 1, (batches,)).to(device)
+ b_seq_start_loc = torch.cumsum(
+ torch.tensor([0] + seq_len_tensor[:-1].tolist(), dtype=torch.long),
+ dim=0,
+ ).to(device)
+ total_tokens = seq_len_tensor.sum()
+
+ lora_weights_lst = []
+ if op_type == "shrink":
+ inputs_tensor = torch.rand((total_tokens, hidden_size), dtype=dtype).to(device)
+
+ for _ in range(nslices):
+ if op_type == "shrink":
+ lora_weights_lst.append(
+ torch.rand(
+ (lora_nums, max_rank, hidden_size), # col-major
+ dtype=dtype,
+ ).to(device)
+ )
+ # NOTE shrink kernel using torch.float32 as output type
+ # shrink op need atomic_add, so output is initinized by 0
+ our_out_tensor = torch.zeros(
+ (nslices, total_tokens, max_rank),
+ dtype=torch.float32,
+ ).to(device)
+ else:
+ inputs_tensor = torch.rand(
+ (nslices, total_tokens, max_rank),
+ dtype=dtype,
+ ).to(device)
+ for _ in range(nslices):
+ lora_weights_lst.append(
+ torch.rand(
+ (lora_nums, hidden_size, max_rank), # col-major
+ dtype=dtype,
+ ).to(device)
+ )
+ # expand op needs to complete y+=a@lora_b, so output is
+ # initinized randomly
+ our_out_tensor = torch.rand(
+ (total_tokens, hidden_size * nslices), dtype=dtype
+ ).to(device)
+
+ # Ensure the same input.
+ ref_out_tensor = our_out_tensor.clone()
+ lora_indices_tensor = torch.randint(
+ 0, lora_nums - 1 if lora_nums > 1 else 1, (batches,)
+ )
+ indices = torch.zeros((total_tokens), dtype=torch.long).to(device)
+ current_offset = 0
+ for b_id in range(batches):
+ lora_index = lora_indices_tensor[b_id]
+ indices[current_offset : current_offset + seq_len_tensor[b_id]] = (
+ lora_index.item()
+ )
+ current_offset += seq_len_tensor[b_id].item()
+
+ lora_indices_tensor = lora_indices_tensor.to(device)
+ return PunicaTensors(
+ inputs_tensor,
+ lora_weights_lst,
+ our_out_tensor,
+ ref_out_tensor,
+ b_seq_start_loc,
+ lora_indices_tensor,
+ seq_len_tensor,
+ indices,
+ )
+
+
+def create_peft_lora(
+ model: torch.nn.Module,
+ save_dir: str,
+ target_modules: list[str],
+ rank: int = 8,
+ alpha: int = 16,
+ dropout: float = 0.1,
+ lora_dtype: torch.dtype = torch.float16,
+) -> dict[str, torch.Tensor]:
+ lora_weights = {}
+ adapter_config = {
+ "peft_type": "LORA",
+ "auto_mapping": None,
+ "base_model_name_or_path": "dummy_model",
+ "revision": None,
+ "task_type": "CAUSAL_LM",
+ "inference_mode": False,
+ "r": rank,
+ "lora_alpha": alpha,
+ "lora_dropout": dropout,
+ "fan_in_fan_out": False,
+ "bias": "none",
+ "modules_to_save": None,
+ "init_lora_weights": True,
+ "layers_to_transform": None,
+ "layers_pattern": None,
+ "target_modules": target_modules,
+ "exclude_modules": None,
+ "use_rslora": False,
+ "use_dora": False,
+ "loftq_config": None,
+ }
+
+ for module_name in target_modules:
+ module = model
+ for attr in module_name.split("."):
+ module = getattr(module, attr)
+
+ if hasattr(module, "input_size") and hasattr(module, "output_size"):
+ in_features = module.input_size
+ out_features = module.output_size
+
+ elif hasattr(module, "embedding_dim") and hasattr(module, "num_embeddings"):
+ # ParallelLMHead
+ in_features = module.embedding_dim
+ out_features = module.num_embeddings
+ else:
+ raise ValueError(f"Unable to determine dimensions for module {module_name}")
+
+ lora_A = torch.randn(rank, in_features, dtype=lora_dtype)
+
+ torch.nn.init.kaiming_uniform_(lora_A, a=5**0.5)
+
+ lora_B = torch.zeros(out_features, rank, dtype=lora_dtype)
+
+ # PEFT style
+ lora_weights[f"base_model.model.{module_name}.lora_A.weight"] = lora_A
+ lora_weights[f"base_model.model.{module_name}.lora_B.weight"] = lora_B
+
+ config_path = os.path.join(save_dir, "adapter_config.json")
+ with open(config_path, "w", encoding="utf-8") as f:
+ json.dump(adapter_config, f, indent=2, ensure_ascii=False)
+
+ weights_path = os.path.join(save_dir, "adapter_model.safetensors")
+ save_file(lora_weights, weights_path)
+
+ return lora_weights
diff --git a/asian_ai/performance/vLLM/tests/model_executor/__init__.py b/asian_ai/performance/vLLM/tests/model_executor/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_activations.py b/asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_activations.py
new file mode 100644
index 0000000000000000000000000000000000000000..a3e0a4ae4082365251cd9f92bcf6e5f9b85bcb58
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_activations.py
@@ -0,0 +1,234 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Unit tests for vllm.model_executor.layers.pooler.activations."""
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+import torch.nn as nn
+
+from vllm.model_executor.layers.pooler.activations import (
+ LambdaPoolerActivation,
+ PoolerClassify,
+ PoolerIdentity,
+ PoolerMultiLabelClassify,
+ PoolerNormalize,
+ get_act_fn,
+ resolve_classifier_act_fn,
+)
+
+
+# ---------------------------------------------------------------------------
+# PoolerIdentity
+# ---------------------------------------------------------------------------
+class TestPoolerIdentity:
+ def test_returns_input_unchanged(self):
+ pooler = PoolerIdentity()
+ x = torch.randn(4, 128)
+ out = pooler(x)
+ assert torch.equal(out, x)
+
+ def test_forward_list(self):
+ pooler = PoolerIdentity()
+ tensors = [torch.randn(128), torch.randn(256)]
+ out = pooler(tensors)
+ assert len(out) == 2
+ for orig, result in zip(tensors, out):
+ assert torch.equal(orig, result)
+
+
+# ---------------------------------------------------------------------------
+# PoolerNormalize
+# ---------------------------------------------------------------------------
+class TestPoolerNormalize:
+ def test_output_has_unit_norm(self):
+ pooler = PoolerNormalize()
+ x = torch.randn(4, 128)
+ out = pooler(x)
+ norms = torch.linalg.norm(out, dim=-1)
+ assert torch.allclose(norms, torch.ones(4), atol=1e-5)
+
+ def test_single_vector(self):
+ pooler = PoolerNormalize()
+ x = torch.randn(1, 64)
+ out = pooler(x)
+ norm = torch.linalg.norm(out, dim=-1)
+ assert torch.allclose(norm, torch.ones(1), atol=1e-5)
+
+ def test_forward_list(self):
+ pooler = PoolerNormalize()
+ tensors = [torch.randn(1, 64), torch.randn(1, 128)]
+ out = pooler(tensors)
+ for t in out:
+ norm = torch.linalg.norm(t, dim=-1)
+ assert torch.allclose(norm, torch.ones(1), atol=1e-5)
+
+
+# ---------------------------------------------------------------------------
+# PoolerMultiLabelClassify
+# ---------------------------------------------------------------------------
+class TestPoolerMultiLabelClassify:
+ def test_output_in_zero_one(self):
+ pooler = PoolerMultiLabelClassify()
+ x = torch.randn(4, 10)
+ out = pooler(x)
+ assert (out >= 0).all() and (out <= 1).all()
+
+ def test_large_positive_maps_near_one(self):
+ pooler = PoolerMultiLabelClassify()
+ x = torch.full((1, 3), 100.0)
+ out = pooler(x)
+ assert torch.allclose(out, torch.ones(1, 3), atol=1e-4)
+
+ def test_large_negative_maps_near_zero(self):
+ pooler = PoolerMultiLabelClassify()
+ x = torch.full((1, 3), -100.0)
+ out = pooler(x)
+ assert torch.allclose(out, torch.zeros(1, 3), atol=1e-4)
+
+
+# ---------------------------------------------------------------------------
+# PoolerClassify
+# ---------------------------------------------------------------------------
+class TestPoolerClassify:
+ def test_infers_from_shape_when_num_labels_none(self):
+ pooler = PoolerClassify(num_labels=None)
+ assert pooler.num_labels is None
+ x = torch.randn(2, 5)
+ out = pooler(x)
+ sums = out.sum(dim=-1)
+ assert torch.allclose(sums, torch.ones(2), atol=1e-5)
+
+ def test_sigmoid_when_num_labels_lt_2(self):
+ pooler = PoolerClassify(num_labels=1)
+ x = torch.zeros(1, 1)
+ out = pooler(x)
+ assert torch.allclose(out, torch.tensor([[0.5]]), atol=1e-5)
+
+ def test_num_labels_zero_uses_sigmoid(self):
+ pooler = PoolerClassify(num_labels=0)
+ assert pooler.num_labels == 0
+ x = torch.zeros(1, 3)
+ out = pooler(x)
+ assert torch.allclose(out, torch.full((1, 3), 0.5), atol=1e-5)
+
+ def test_num_labels_ge_2_uses_softmax(self):
+ pooler = PoolerClassify(num_labels=4)
+ assert pooler.num_labels == 4
+ x = torch.randn(2, 4)
+ out = pooler(x)
+ sums = out.sum(dim=-1)
+ assert torch.allclose(sums, torch.ones(2), atol=1e-5)
+
+ def test_default_num_labels_is_none(self):
+ pooler = PoolerClassify()
+ assert pooler.num_labels is None
+
+
+# ---------------------------------------------------------------------------
+# LambdaPoolerActivation
+# ---------------------------------------------------------------------------
+class TestLambdaPoolerActivation:
+ def test_applies_custom_fn(self):
+ pooler = LambdaPoolerActivation(nn.ReLU())
+ x = torch.tensor([[-1.0, 2.0, -3.0]])
+ out = pooler(x)
+ expected = torch.tensor([[0.0, 2.0, 0.0]])
+ assert torch.equal(out, expected)
+
+ def test_forward_list(self):
+ pooler = LambdaPoolerActivation(nn.ReLU())
+ tensors = [torch.tensor([-1.0, 2.0]), torch.tensor([3.0, -4.0])]
+ out = pooler(tensors)
+ assert torch.equal(out[0], torch.tensor([0.0, 2.0]))
+ assert torch.equal(out[1], torch.tensor([3.0, 0.0]))
+
+
+# ---------------------------------------------------------------------------
+# get_act_fn factory
+# ---------------------------------------------------------------------------
+class TestGetActFn:
+ @staticmethod
+ def _make_config(**kwargs):
+ return SimpleNamespace(**kwargs)
+
+ def test_regression(self):
+ cfg = self._make_config(problem_type="regression")
+ result = get_act_fn(cfg)
+ assert isinstance(result, PoolerIdentity)
+
+ def test_single_label_classification(self):
+ cfg = self._make_config(
+ problem_type="single_label_classification", num_labels=3
+ )
+ result = get_act_fn(cfg)
+ assert isinstance(result, PoolerClassify)
+ assert result.num_labels == 3
+
+ def test_multi_label_classification(self):
+ cfg = self._make_config(problem_type="multi_label_classification")
+ result = get_act_fn(cfg)
+ assert isinstance(result, PoolerMultiLabelClassify)
+
+ def test_sentence_transformers_activation(self):
+ cfg = self._make_config(
+ problem_type="",
+ sentence_transformers={
+ "activation_fn": "torch.nn.modules.activation.Sigmoid"
+ },
+ )
+ result = get_act_fn(cfg)
+ assert isinstance(result, PoolerClassify)
+
+ def test_sbert_activation(self):
+ cfg = self._make_config(
+ problem_type="",
+ sbert_ce_default_activation_function=(
+ "torch.nn.modules.activation.Sigmoid"
+ ),
+ )
+ result = get_act_fn(cfg)
+ assert isinstance(result, PoolerClassify)
+
+ def test_default_fallback(self):
+ cfg = self._make_config(problem_type="")
+ result = get_act_fn(cfg)
+ assert isinstance(result, PoolerClassify)
+
+ def test_sentence_transformers_takes_priority(self):
+ cfg = self._make_config(
+ problem_type="",
+ sentence_transformers={"activation_fn": "torch.nn.modules.linear.Identity"},
+ sbert_ce_default_activation_function=(
+ "torch.nn.modules.activation.Sigmoid"
+ ),
+ )
+ result = get_act_fn(cfg)
+ assert isinstance(result, PoolerIdentity)
+
+ def test_rejects_non_torch_activation(self):
+ cfg = self._make_config(
+ problem_type="",
+ sentence_transformers={"activation_fn": "os.system"},
+ )
+ with pytest.raises(ValueError, match="restricted"):
+ get_act_fn(cfg)
+
+
+# ---------------------------------------------------------------------------
+# resolve_classifier_act_fn
+# ---------------------------------------------------------------------------
+class TestResolveClassifierActFn:
+ def test_delegates_to_get_act_fn_when_none(self):
+ model_config = SimpleNamespace(
+ hf_config=SimpleNamespace(num_labels=3, problem_type="")
+ )
+ result = resolve_classifier_act_fn(model_config, act_fn=None)
+ assert isinstance(result, PoolerClassify)
+ assert result.num_labels == 3
+
+ def test_passes_through_provided_act_fn(self):
+ custom = PoolerIdentity()
+ result = resolve_classifier_act_fn(None, act_fn=custom)
+ assert result is custom
diff --git a/asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_heads.py b/asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_heads.py
new file mode 100644
index 0000000000000000000000000000000000000000..479c3fe181b5f2e4b8e53fbe20b06d95885ad258
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_heads.py
@@ -0,0 +1,481 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Unit tests for sequence and token pooler head classes."""
+
+import torch
+import torch.nn as nn
+
+from vllm.model_executor.layers.pooler.activations import PoolerNormalize
+from vllm.model_executor.layers.pooler.seqwise.heads import (
+ ClassifierPoolerHead,
+ EmbeddingPoolerHead,
+)
+from vllm.model_executor.layers.pooler.tokwise.heads import (
+ TokenClassifierPoolerHead,
+ TokenEmbeddingPoolerHead,
+)
+from vllm.pooling_params import PoolingParams
+from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
+
+_HIDDEN = 16
+_BATCH = 3
+
+
+def _make_params(
+ n: int,
+ *,
+ task: str = "embed",
+ dimensions: int | None = None,
+ use_activation: bool | None = None,
+) -> list[PoolingParams]:
+ return [
+ PoolingParams(task=task, dimensions=dimensions, use_activation=use_activation)
+ for _ in range(n)
+ ]
+
+
+def _make_metadata(pooling_params: list[PoolingParams]) -> PoolingMetadata:
+ n = len(pooling_params)
+ return PoolingMetadata(
+ prompt_lens=torch.ones(n, dtype=torch.long),
+ prompt_token_ids=None,
+ prompt_token_ids_cpu=None,
+ pooling_params=pooling_params,
+ pooling_states=[PoolingStates() for _ in range(n)],
+ )
+
+
+def _linear(in_f: int, out_f: int) -> nn.Linear:
+ torch.manual_seed(42)
+ return nn.Linear(in_f, out_f, bias=False)
+
+
+# ---------------------------------------------------------------------------
+# EmbeddingPoolerHead
+# ---------------------------------------------------------------------------
+class TestEmbeddingPoolerHead:
+ def test_supported_tasks(self):
+ head = EmbeddingPoolerHead()
+ assert head.get_supported_tasks() == {"embed"}
+
+ def test_passthrough(self):
+ head = EmbeddingPoolerHead()
+ x = torch.randn(_BATCH, _HIDDEN)
+ meta = _make_metadata(_make_params(_BATCH))
+ out = head(x, meta)
+ assert torch.equal(out, x)
+
+ def test_head_dtype(self):
+ head = EmbeddingPoolerHead(head_dtype=torch.float16)
+ x = torch.randn(_BATCH, _HIDDEN)
+ meta = _make_metadata(_make_params(_BATCH))
+ out = head(x, meta)
+ assert out.dtype == torch.float16
+
+ def test_projector(self):
+ proj = _linear(_HIDDEN, 8)
+ head = EmbeddingPoolerHead(projector=proj)
+ x = torch.randn(_BATCH, _HIDDEN)
+ meta = _make_metadata(_make_params(_BATCH))
+ out = head(x, meta)
+ assert out.shape == (_BATCH, 8)
+ assert torch.allclose(out, proj(x))
+
+ def test_matryoshka_uniform(self):
+ head = EmbeddingPoolerHead()
+ x = torch.randn(_BATCH, _HIDDEN)
+ params = _make_params(_BATCH, dimensions=4)
+ meta = _make_metadata(params)
+ out = head(x, meta)
+ assert out.shape == (_BATCH, 4)
+ assert torch.equal(out, x[..., :4])
+
+ def test_matryoshka_mixed(self):
+ head = EmbeddingPoolerHead()
+ x = torch.randn(2, _HIDDEN)
+ params = [
+ PoolingParams(task="embed", dimensions=4),
+ PoolingParams(task="embed", dimensions=8),
+ ]
+ meta = _make_metadata(params)
+ out = head(x, meta)
+ assert isinstance(out, list)
+ assert len(out) == 2
+ assert out[0].shape[-1] == 4
+ assert out[1].shape[-1] == 8
+
+ def test_matryoshka_mixed_with_none(self):
+ head = EmbeddingPoolerHead()
+ x = torch.randn(2, _HIDDEN)
+ params = [
+ PoolingParams(task="embed", dimensions=4),
+ PoolingParams(task="embed", dimensions=None),
+ ]
+ meta = _make_metadata(params)
+ out = head(x, meta)
+ assert isinstance(out, list)
+ assert out[0].shape[-1] == 4
+ assert torch.equal(out[1], x[1])
+
+ def test_activation_uniform_true(self):
+ head = EmbeddingPoolerHead(activation=PoolerNormalize())
+ x = torch.randn(_BATCH, _HIDDEN)
+ params = _make_params(_BATCH, use_activation=True)
+ meta = _make_metadata(params)
+ out = head(x, meta)
+ norms = torch.linalg.norm(out, dim=-1)
+ assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
+
+ def test_activation_uniform_false(self):
+ head = EmbeddingPoolerHead(activation=PoolerNormalize())
+ x = torch.randn(_BATCH, _HIDDEN)
+ params = _make_params(_BATCH, use_activation=False)
+ meta = _make_metadata(params)
+ out = head(x, meta)
+ assert torch.equal(out, x)
+
+ def test_activation_mixed_flags(self):
+ head = EmbeddingPoolerHead(activation=PoolerNormalize())
+ x = torch.randn(2, _HIDDEN)
+ params = [
+ PoolingParams(task="embed", use_activation=True),
+ PoolingParams(task="embed", use_activation=False),
+ ]
+ meta = _make_metadata(params)
+ out = head(x, meta)
+ assert isinstance(out, list)
+ norm_0 = torch.linalg.norm(out[0], dim=-1)
+ assert torch.allclose(norm_0, torch.ones(1), atol=1e-5)
+ assert torch.equal(out[1], x[1])
+
+ def test_list_input_gets_stacked(self):
+ head = EmbeddingPoolerHead()
+ tensors = [torch.randn(_HIDDEN) for _ in range(_BATCH)]
+ meta = _make_metadata(_make_params(_BATCH))
+ out = head(tensors, meta)
+ assert out.shape == (_BATCH, _HIDDEN)
+ expected = torch.stack(tensors)
+ assert torch.equal(out, expected)
+
+ def test_projector_then_matryoshka(self):
+ proj = _linear(_HIDDEN, 8)
+ head = EmbeddingPoolerHead(projector=proj)
+ x = torch.randn(_BATCH, _HIDDEN)
+ params = _make_params(_BATCH, dimensions=4)
+ meta = _make_metadata(params)
+ out = head(x, meta)
+ assert out.shape == (_BATCH, 4)
+ assert torch.equal(out, proj(x)[..., :4])
+
+ def test_matryoshka_then_activation(self):
+ head = EmbeddingPoolerHead(activation=PoolerNormalize())
+ x = torch.randn(_BATCH, _HIDDEN)
+ params = _make_params(_BATCH, dimensions=4, use_activation=True)
+ meta = _make_metadata(params)
+ out = head(x, meta)
+ assert out.shape == (_BATCH, 4)
+ norms = torch.linalg.norm(out, dim=-1)
+ assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
+
+ def test_empty_batch(self):
+ head = EmbeddingPoolerHead()
+ x = torch.randn(0, _HIDDEN)
+ meta = _make_metadata([])
+ out = head(x, meta)
+ assert out.shape == (0, _HIDDEN)
+
+
+# ---------------------------------------------------------------------------
+# ClassifierPoolerHead
+# ---------------------------------------------------------------------------
+class TestClassifierPoolerHead:
+ def test_supported_tasks(self):
+ head = ClassifierPoolerHead()
+ assert head.get_supported_tasks() == {"classify"}
+
+ def test_passthrough(self):
+ head = ClassifierPoolerHead()
+ x = torch.randn(_BATCH, _HIDDEN)
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
+ out = head(x, meta)
+ assert torch.equal(out, x)
+
+ def test_head_dtype(self):
+ head = ClassifierPoolerHead(head_dtype=torch.float16)
+ x = torch.randn(_BATCH, _HIDDEN)
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
+ out = head(x, meta)
+ assert out.dtype == torch.float16
+
+ def test_classifier(self):
+ clf = _linear(_HIDDEN, 3)
+ head = ClassifierPoolerHead(classifier=clf)
+ x = torch.randn(_BATCH, _HIDDEN)
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
+ out = head(x, meta)
+ assert out.shape == (_BATCH, 3)
+ assert torch.allclose(out, clf(x))
+
+ def test_logit_mean(self):
+ head = ClassifierPoolerHead(logit_mean=2.0)
+ x = torch.randn(_BATCH, _HIDDEN)
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
+ out = head(x, meta)
+ assert torch.allclose(out, x - 2.0)
+
+ def test_logit_sigma(self):
+ head = ClassifierPoolerHead(logit_sigma=0.5)
+ x = torch.randn(_BATCH, _HIDDEN)
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
+ out = head(x, meta)
+ assert torch.allclose(out, x / 0.5)
+
+ def test_platt_scaling_combined(self):
+ head = ClassifierPoolerHead(logit_mean=1.0, logit_sigma=2.0)
+ x = torch.randn(_BATCH, _HIDDEN)
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
+ out = head(x, meta)
+ assert torch.allclose(out, (x - 1.0) / 2.0)
+
+ def test_activation_uniform_true(self):
+ head = ClassifierPoolerHead(activation=PoolerNormalize())
+ x = torch.randn(_BATCH, _HIDDEN)
+ params = _make_params(_BATCH, task="classify", use_activation=True)
+ meta = _make_metadata(params)
+ out = head(x, meta)
+ norms = torch.linalg.norm(out, dim=-1)
+ assert torch.allclose(norms, torch.ones(_BATCH), atol=1e-5)
+
+ def test_activation_uniform_false(self):
+ head = ClassifierPoolerHead(activation=PoolerNormalize())
+ x = torch.randn(_BATCH, _HIDDEN)
+ params = _make_params(_BATCH, task="classify", use_activation=False)
+ meta = _make_metadata(params)
+ out = head(x, meta)
+ assert torch.equal(out, x)
+
+ def test_activation_mixed_flags(self):
+ head = ClassifierPoolerHead(activation=PoolerNormalize())
+ x = torch.randn(2, _HIDDEN)
+ params = [
+ PoolingParams(task="classify", use_activation=True),
+ PoolingParams(task="classify", use_activation=False),
+ ]
+ meta = _make_metadata(params)
+ out = head(x, meta)
+ assert isinstance(out, list)
+ norm_0 = torch.linalg.norm(out[0], dim=-1)
+ assert torch.allclose(norm_0, torch.ones(1), atol=1e-5)
+ assert torch.equal(out[1], x[1])
+
+ def test_list_input_gets_stacked(self):
+ head = ClassifierPoolerHead()
+ tensors = [torch.randn(_HIDDEN) for _ in range(_BATCH)]
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
+ out = head(tensors, meta)
+ assert out.shape == (_BATCH, _HIDDEN)
+ expected = torch.stack(tensors)
+ assert torch.equal(out, expected)
+
+ def test_classifier_then_platt_scaling(self):
+ clf = _linear(_HIDDEN, 3)
+ head = ClassifierPoolerHead(classifier=clf, logit_mean=1.0, logit_sigma=2.0)
+ x = torch.randn(_BATCH, _HIDDEN)
+ meta = _make_metadata(_make_params(_BATCH, task="classify"))
+ out = head(x, meta)
+ expected = (clf(x) - 1.0) / 2.0
+ assert torch.allclose(out, expected)
+
+ def test_empty_batch(self):
+ head = ClassifierPoolerHead()
+ x = torch.randn(0, _HIDDEN)
+ meta = _make_metadata([])
+ out = head(x, meta)
+ assert out.shape == (0, _HIDDEN)
+
+
+# ---------------------------------------------------------------------------
+# TokenEmbeddingPoolerHead
+# ---------------------------------------------------------------------------
+class TestTokenEmbeddingPoolerHead:
+ def test_supported_tasks(self):
+ head = TokenEmbeddingPoolerHead()
+ assert head.get_supported_tasks() == {"token_embed"}
+
+ def test_passthrough(self):
+ head = TokenEmbeddingPoolerHead()
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_embed")
+ out = head.forward_chunk(x, param)
+ assert torch.equal(out, x)
+
+ def test_none_chunked_prefill(self):
+ head = TokenEmbeddingPoolerHead()
+ param = PoolingParams(task="token_embed")
+ out = head.forward_chunk(None, param)
+ assert out is None
+
+ def test_head_dtype(self):
+ head = TokenEmbeddingPoolerHead(head_dtype=torch.float16)
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_embed")
+ out = head.forward_chunk(x, param)
+ assert out.dtype == torch.float16
+
+ def test_projector(self):
+ proj = _linear(_HIDDEN, 8)
+ head = TokenEmbeddingPoolerHead(projector=proj)
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_embed")
+ out = head.forward_chunk(x, param)
+ assert out.shape == (5, 8)
+ assert torch.allclose(out, proj(x))
+
+ def test_matryoshka_truncation(self):
+ head = TokenEmbeddingPoolerHead()
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_embed", dimensions=4)
+ out = head.forward_chunk(x, param)
+ assert out.shape == (5, 4)
+ assert torch.equal(out, x[..., :4])
+
+ def test_activation_true(self):
+ head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_embed", use_activation=True)
+ out = head.forward_chunk(x, param)
+ norms = torch.linalg.norm(out, dim=-1)
+ assert torch.allclose(norms, torch.ones(5), atol=1e-5)
+
+ def test_activation_false(self):
+ head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_embed", use_activation=False)
+ out = head.forward_chunk(x, param)
+ assert torch.equal(out, x)
+
+ def test_projector_then_matryoshka(self):
+ proj = _linear(_HIDDEN, 8)
+ head = TokenEmbeddingPoolerHead(projector=proj)
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_embed", dimensions=4)
+ out = head.forward_chunk(x, param)
+ assert out.shape == (5, 4)
+ assert torch.equal(out, proj(x)[..., :4])
+
+ def test_matryoshka_then_activation(self):
+ head = TokenEmbeddingPoolerHead(activation=PoolerNormalize())
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_embed", dimensions=4, use_activation=True)
+ out = head.forward_chunk(x, param)
+ assert out.shape == (5, 4)
+ norms = torch.linalg.norm(out, dim=-1)
+ assert torch.allclose(norms, torch.ones(5), atol=1e-5)
+
+ def test_forward_mixed_batch_chunked_prefill(self):
+ head = TokenEmbeddingPoolerHead()
+ pooled_data = [torch.randn(5, _HIDDEN), None, torch.randn(3, _HIDDEN)]
+ params = _make_params(3, task="token_embed")
+ meta = _make_metadata(params)
+ out = head(pooled_data, meta)
+ assert len(out) == 3
+ assert torch.equal(out[0], pooled_data[0])
+ assert out[1] is None
+ assert torch.equal(out[2], pooled_data[2])
+
+ def test_forward_empty_batch(self):
+ head = TokenEmbeddingPoolerHead()
+ meta = _make_metadata([])
+ out = head([], meta)
+ assert out == []
+
+
+# ---------------------------------------------------------------------------
+# TokenClassifierPoolerHead
+# ---------------------------------------------------------------------------
+class TestTokenClassifierPoolerHead:
+ def test_supported_tasks(self):
+ head = TokenClassifierPoolerHead()
+ assert head.get_supported_tasks() == {"token_classify"}
+
+ def test_passthrough(self):
+ head = TokenClassifierPoolerHead()
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_classify")
+ out = head.forward_chunk(x, param)
+ assert torch.equal(out, x)
+
+ def test_none_chunked_prefill(self):
+ head = TokenClassifierPoolerHead()
+ param = PoolingParams(task="token_classify")
+ out = head.forward_chunk(None, param)
+ assert out is None
+
+ def test_head_dtype(self):
+ head = TokenClassifierPoolerHead(head_dtype=torch.float16)
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_classify")
+ out = head.forward_chunk(x, param)
+ assert out.dtype == torch.float16
+
+ def test_classifier(self):
+ clf = _linear(_HIDDEN, 3)
+ head = TokenClassifierPoolerHead(classifier=clf)
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_classify")
+ out = head.forward_chunk(x, param)
+ assert out.shape == (5, 3)
+ assert torch.allclose(out, clf(x))
+
+ def test_logit_mean(self):
+ head = TokenClassifierPoolerHead(logit_mean=2.0)
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_classify")
+ out = head.forward_chunk(x, param)
+ assert torch.allclose(out, x - 2.0)
+
+ def test_logit_sigma(self):
+ head = TokenClassifierPoolerHead(logit_sigma=0.5)
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_classify")
+ out = head.forward_chunk(x, param)
+ assert torch.allclose(out, x / 0.5)
+
+ def test_platt_scaling_combined(self):
+ head = TokenClassifierPoolerHead(logit_mean=1.0, logit_sigma=2.0)
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_classify")
+ out = head.forward_chunk(x, param)
+ assert torch.allclose(out, (x - 1.0) / 2.0)
+
+ def test_activation_true(self):
+ head = TokenClassifierPoolerHead(activation=PoolerNormalize())
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_classify", use_activation=True)
+ out = head.forward_chunk(x, param)
+ norms = torch.linalg.norm(out, dim=-1)
+ assert torch.allclose(norms, torch.ones(5), atol=1e-5)
+
+ def test_activation_false(self):
+ head = TokenClassifierPoolerHead(activation=PoolerNormalize())
+ x = torch.randn(5, _HIDDEN)
+ param = PoolingParams(task="token_classify", use_activation=False)
+ out = head.forward_chunk(x, param)
+ assert torch.equal(out, x)
+
+ def test_forward_mixed_batch_chunked_prefill(self):
+ head = TokenClassifierPoolerHead()
+ pooled_data = [torch.randn(5, _HIDDEN), None, torch.randn(3, _HIDDEN)]
+ params = _make_params(3, task="token_classify")
+ meta = _make_metadata(params)
+ out = head(pooled_data, meta)
+ assert len(out) == 3
+ assert torch.equal(out[0], pooled_data[0])
+ assert out[1] is None
+ assert torch.equal(out[2], pooled_data[2])
+
+ def test_forward_empty_batch(self):
+ head = TokenClassifierPoolerHead()
+ meta = _make_metadata([])
+ out = head([], meta)
+ assert out == []
diff --git a/asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_methods.py b/asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_methods.py
new file mode 100644
index 0000000000000000000000000000000000000000..e6e567fe6787bd3a882fb1eaaa921404f8d6755f
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/layers/test_pooler_methods.py
@@ -0,0 +1,499 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Unit tests for sequence and token pooling methods and their factories."""
+
+from dataclasses import dataclass
+from unittest.mock import patch
+
+import pytest
+import torch
+
+from vllm.model_executor.layers.pooler.seqwise.methods import (
+ CLSPool,
+ LastPool,
+ MeanPool,
+ get_seq_pooling_method,
+)
+from vllm.model_executor.layers.pooler.tokwise.methods import (
+ AllPool,
+ StepPool,
+ get_tok_pooling_method,
+)
+from vllm.pooling_params import PoolingParams
+from vllm.v1.pool.metadata import PoolingCursor, PoolingMetadata, PoolingStates
+
+_CPU = torch.device("cpu")
+
+
+def _make_pooling_cursor(
+ prompt_lens: list[int],
+ *,
+ num_scheduled_tokens: list[int] | None = None,
+ seq_lens: list[int] | None = None,
+ device: torch.device = _CPU,
+) -> PoolingCursor:
+ """Build a PoolingCursor from a list of per-sequence prompt lengths."""
+ prompt_lens_cpu = torch.tensor(prompt_lens, dtype=torch.long)
+ if num_scheduled_tokens is None:
+ num_scheduled_tokens_cpu = prompt_lens_cpu.clone()
+ else:
+ num_scheduled_tokens_cpu = torch.tensor(num_scheduled_tokens, dtype=torch.long)
+ if seq_lens is None:
+ seq_lens_cpu = prompt_lens_cpu.clone()
+ else:
+ seq_lens_cpu = torch.tensor(seq_lens, dtype=torch.long)
+
+ cumsum = torch.zeros(len(prompt_lens) + 1, dtype=torch.long, device=device)
+ torch.cumsum(num_scheduled_tokens_cpu, dim=0, out=cumsum[1:])
+
+ return PoolingCursor(
+ first_token_indices_gpu=cumsum[: len(prompt_lens)].to(device),
+ last_token_indices_gpu=(cumsum[1:] - 1).to(device),
+ prompt_lens_cpu=prompt_lens_cpu,
+ seq_lens_cpu=seq_lens_cpu,
+ num_scheduled_tokens_cpu=num_scheduled_tokens_cpu,
+ )
+
+
+def _make_metadata(
+ prompt_lens: list[int],
+ *,
+ tasks: list[str] | None = None,
+ token_ids: list[list[int]] | None = None,
+ pooling_params: list[PoolingParams] | None = None,
+ num_scheduled_tokens: list[int] | None = None,
+ seq_lens: list[int] | None = None,
+ device: torch.device = _CPU,
+) -> PoolingMetadata:
+ """Build a minimal PoolingMetadata for testing pooling methods."""
+ n_seqs = len(prompt_lens)
+ if tasks is None:
+ tasks = ["embed"] * n_seqs
+ if pooling_params is None:
+ pooling_params = [PoolingParams(task=t) for t in tasks]
+
+ prompt_lens_tensor = torch.tensor(prompt_lens, dtype=torch.long)
+
+ prompt_token_ids_cpu = None
+ prompt_token_ids = None
+ if token_ids is not None:
+ max_len = max(len(t) for t in token_ids)
+ padded = [t + [0] * (max_len - len(t)) for t in token_ids]
+ prompt_token_ids_cpu = torch.tensor(padded, dtype=torch.long)
+ prompt_token_ids = prompt_token_ids_cpu.to(device)
+
+ cursor = _make_pooling_cursor(
+ prompt_lens,
+ num_scheduled_tokens=num_scheduled_tokens,
+ seq_lens=seq_lens,
+ device=device,
+ )
+
+ pooling_states = [PoolingStates() for _ in range(n_seqs)]
+
+ return PoolingMetadata(
+ prompt_lens=prompt_lens_tensor,
+ prompt_token_ids=prompt_token_ids,
+ prompt_token_ids_cpu=prompt_token_ids_cpu,
+ pooling_params=pooling_params,
+ pooling_states=pooling_states,
+ pooling_cursor=cursor,
+ )
+
+
+# ---------------------------------------------------------------------------
+# CLSPool
+# ---------------------------------------------------------------------------
+class TestCLSPool:
+ def test_extracts_first_token(self):
+ hidden = torch.tensor(
+ [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
+ )
+ metadata = _make_metadata([2, 3])
+ pooler = CLSPool()
+ out = pooler(hidden, metadata)
+ expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
+ assert torch.equal(out, expected)
+
+ def test_rejects_partial_prefill(self):
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
+ metadata = _make_metadata([3], num_scheduled_tokens=[2])
+ pooler = CLSPool()
+ with pytest.raises(RuntimeError, match="partial prefill"):
+ pooler(hidden, metadata)
+
+
+# ---------------------------------------------------------------------------
+# LastPool
+# ---------------------------------------------------------------------------
+class TestLastPool:
+ def test_extracts_last_token(self):
+ hidden = torch.tensor(
+ [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
+ )
+ metadata = _make_metadata([2, 3])
+ pooler = LastPool()
+ out = pooler(hidden, metadata)
+ expected = torch.tensor([[3.0, 4.0], [9.0, 10.0]])
+ assert torch.equal(out, expected)
+
+ def test_partial_prefill_extracts_last_scheduled(self):
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
+ metadata = _make_metadata([4], num_scheduled_tokens=[2])
+ pooler = LastPool()
+ out = pooler(hidden, metadata)
+ expected = torch.tensor([[3.0, 4.0]])
+ assert torch.equal(out, expected)
+
+
+# ---------------------------------------------------------------------------
+# MeanPool
+# ---------------------------------------------------------------------------
+class TestMeanPool:
+ def test_computes_mean(self):
+ hidden = torch.tensor(
+ [[1.0, 2.0], [3.0, 4.0], [10.0, 20.0]], dtype=torch.float32
+ )
+ metadata = _make_metadata([2, 1])
+ pooler = MeanPool()
+ out = pooler(hidden, metadata)
+ expected = torch.tensor([[2.0, 3.0], [10.0, 20.0]], dtype=torch.float32)
+ assert torch.allclose(out, expected, atol=1e-5)
+
+ def test_single_token_is_identity(self):
+ hidden = torch.tensor([[5.0, 10.0]], dtype=torch.float32)
+ metadata = _make_metadata([1])
+ pooler = MeanPool()
+ out = pooler(hidden, metadata)
+ assert torch.allclose(out, hidden, atol=1e-5)
+
+ def test_uniform_values_return_same(self):
+ hidden = torch.full((4, 3), 7.0, dtype=torch.float32)
+ metadata = _make_metadata([4])
+ pooler = MeanPool()
+ out = pooler(hidden, metadata)
+ expected = torch.full((1, 3), 7.0, dtype=torch.float32)
+ assert torch.allclose(out, expected, atol=1e-5)
+
+ def test_multiple_sequences(self):
+ hidden = torch.tensor(
+ [
+ [0.0, 0.0],
+ [2.0, 4.0],
+ [4.0, 8.0],
+ [10.0, 10.0],
+ ],
+ dtype=torch.float32,
+ )
+ metadata = _make_metadata([3, 1])
+ pooler = MeanPool()
+ out = pooler(hidden, metadata)
+ expected = torch.tensor([[2.0, 4.0], [10.0, 10.0]], dtype=torch.float32)
+ assert torch.allclose(out, expected, atol=1e-5)
+
+ def test_empty_batch(self):
+ hidden = torch.empty((0, 8), dtype=torch.float32)
+ metadata = _make_metadata([])
+ pooler = MeanPool()
+ out = pooler(hidden, metadata)
+ assert out.shape == (0, 8)
+
+ def test_rejects_partial_prefill(self):
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float32)
+ metadata = _make_metadata([3], num_scheduled_tokens=[2])
+ pooler = MeanPool()
+ with pytest.raises(RuntimeError, match="partial prefill"):
+ pooler(hidden, metadata)
+
+ def test_chunked_accumulation(self):
+ hidden = torch.arange(20, dtype=torch.float32).reshape(5, 4)
+ metadata = _make_metadata([3, 2])
+ pooler = MeanPool()
+ with patch(
+ "vllm.model_executor.layers.pooler.seqwise.methods"
+ "._MEAN_POOL_ACCUMULATION_CHUNK_BYTES",
+ 16,
+ ):
+ out = pooler(hidden, metadata)
+ expected_seq0 = hidden[:3].float().mean(dim=0, keepdim=True)
+ expected_seq1 = hidden[3:].float().mean(dim=0, keepdim=True)
+ expected = torch.cat([expected_seq0, expected_seq1], dim=0)
+ assert torch.allclose(out, expected, atol=1e-5)
+
+ def test_upcasts_to_float32(self):
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]], dtype=torch.float16)
+ metadata = _make_metadata([2])
+ pooler = MeanPool()
+ out = pooler(hidden, metadata)
+ assert out.dtype == torch.float32
+ expected = torch.tensor([[2.0, 3.0]], dtype=torch.float32)
+ assert torch.allclose(out, expected, atol=1e-2)
+
+
+# ---------------------------------------------------------------------------
+# get_seq_pooling_method factory
+# ---------------------------------------------------------------------------
+class TestGetSeqPoolingMethod:
+ def test_cls(self):
+ assert isinstance(get_seq_pooling_method("CLS"), CLSPool)
+
+ def test_last(self):
+ assert isinstance(get_seq_pooling_method("LAST"), LastPool)
+
+ def test_mean(self):
+ assert isinstance(get_seq_pooling_method("MEAN"), MeanPool)
+
+ def test_unknown_raises(self):
+ with pytest.raises(NotImplementedError, match="UNKNOWN"):
+ get_seq_pooling_method("UNKNOWN")
+
+
+# ---------------------------------------------------------------------------
+# AllPool
+# ---------------------------------------------------------------------------
+
+
+@dataclass
+class _FakeSchedulerConfig:
+ enable_chunked_prefill: bool = False
+
+
+@dataclass
+class _FakeVllmConfig:
+ scheduler_config: _FakeSchedulerConfig
+
+
+class TestAllPool:
+ @staticmethod
+ def _make_all_pool(*, chunked: bool = False) -> AllPool:
+ fake_config = _FakeVllmConfig(
+ scheduler_config=_FakeSchedulerConfig(
+ enable_chunked_prefill=chunked,
+ ),
+ )
+ with patch(
+ "vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
+ return_value=fake_config,
+ ):
+ return AllPool()
+
+ def test_splits_by_sequence(self):
+ pooler = self._make_all_pool()
+ hidden = torch.tensor(
+ [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
+ )
+ metadata = _make_metadata([2, 3])
+ out = pooler(hidden, metadata)
+ assert len(out) == 2
+ assert torch.equal(out[0], hidden[:2])
+ assert torch.equal(out[1], hidden[2:])
+
+ def test_single_sequence(self):
+ pooler = self._make_all_pool()
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
+ metadata = _make_metadata([3])
+ out = pooler(hidden, metadata)
+ assert len(out) == 1
+ assert torch.equal(out[0], hidden)
+
+ def test_chunked_prefill_returns_none_for_unfinished(self):
+ pooler = self._make_all_pool(chunked=True)
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
+ metadata = _make_metadata(
+ [4],
+ num_scheduled_tokens=[2],
+ seq_lens=[2],
+ )
+ out = pooler(hidden, metadata)
+ assert len(out) == 1
+ assert out[0] is None
+
+ def test_chunked_prefill_returns_concat_when_finished(self):
+ pooler = self._make_all_pool(chunked=True)
+
+ chunk1 = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
+ metadata1 = _make_metadata(
+ [4],
+ num_scheduled_tokens=[2],
+ seq_lens=[2],
+ )
+ out1 = pooler(chunk1, metadata1)
+ assert out1[0] is None
+
+ chunk2 = torch.tensor([[5.0, 6.0], [7.0, 8.0]])
+ metadata2 = _make_metadata(
+ [4],
+ num_scheduled_tokens=[2],
+ seq_lens=[4],
+ )
+ metadata2.pooling_states = metadata1.pooling_states
+ out2 = pooler(chunk2, metadata2)
+ assert out2[0] is not None
+ expected = torch.cat([chunk1, chunk2], dim=0)
+ assert torch.equal(out2[0], expected)
+
+ def test_chunked_prefill_single_shot_matches_non_chunked(self):
+ pooler = self._make_all_pool(chunked=True)
+ hidden = torch.tensor(
+ [[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0], [9.0, 10.0]]
+ )
+ metadata = _make_metadata([2, 3])
+ out = pooler(hidden, metadata)
+ assert len(out) == 2
+ assert torch.equal(out[0], hidden[:2])
+ assert torch.equal(out[1], hidden[2:])
+
+ def test_chunked_prefill_mixed_finished_unfinished(self):
+ pooler = self._make_all_pool(chunked=True)
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0]])
+ metadata = _make_metadata(
+ [2, 4],
+ num_scheduled_tokens=[2, 1],
+ seq_lens=[2, 1],
+ )
+ out = pooler(hidden, metadata)
+ assert len(out) == 2
+ assert torch.equal(out[0], hidden[:2])
+ assert out[1] is None
+
+
+# ---------------------------------------------------------------------------
+# StepPool
+# ---------------------------------------------------------------------------
+class TestStepPool:
+ @staticmethod
+ def _make_step_pool(*, chunked: bool = False) -> StepPool:
+ fake_config = _FakeVllmConfig(
+ scheduler_config=_FakeSchedulerConfig(
+ enable_chunked_prefill=chunked,
+ ),
+ )
+ with patch(
+ "vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
+ return_value=fake_config,
+ ):
+ return StepPool()
+
+ def test_filters_by_step_tag_id(self):
+ pooler = self._make_step_pool()
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
+ token_ids = [[10, 99, 10, 20]]
+ params = [PoolingParams(task="token_classify", step_tag_id=10)]
+ metadata = _make_metadata([4], token_ids=token_ids, pooling_params=params)
+ out = pooler(hidden, metadata)
+ assert len(out) == 1
+ expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
+ assert torch.equal(out[0], expected)
+
+ def test_filters_by_returned_token_ids(self):
+ pooler = self._make_step_pool()
+ hidden = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]])
+ token_ids = [[10, 20]]
+ params = [PoolingParams(task="token_classify", returned_token_ids=[0, 2])]
+ metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
+ out = pooler(hidden, metadata)
+ assert len(out) == 1
+ expected = torch.tensor([[1.0, 3.0], [4.0, 6.0]])
+ assert torch.equal(out[0], expected)
+
+ def test_no_filtering_without_params(self):
+ pooler = self._make_step_pool()
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
+ token_ids = [[10, 20]]
+ params = [PoolingParams(task="token_classify")]
+ metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
+ out = pooler(hidden, metadata)
+ assert len(out) == 1
+ assert torch.equal(out[0], hidden)
+
+ def test_combined_step_tag_and_returned_token_ids(self):
+ pooler = self._make_step_pool()
+ hidden = torch.tensor([[1.0, 2.0, 3.0], [4.0, 5.0, 6.0], [7.0, 8.0, 9.0]])
+ token_ids = [[99, 10, 99]]
+ params = [
+ PoolingParams(
+ task="token_classify",
+ step_tag_id=10,
+ returned_token_ids=[0, 2],
+ )
+ ]
+ metadata = _make_metadata([3], token_ids=token_ids, pooling_params=params)
+ out = pooler(hidden, metadata)
+ assert len(out) == 1
+ expected = torch.tensor([[4.0, 6.0]])
+ assert torch.equal(out[0], expected)
+
+ def test_step_tag_id_no_match_returns_empty(self):
+ pooler = self._make_step_pool()
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
+ token_ids = [[10, 20]]
+ params = [PoolingParams(task="token_classify", step_tag_id=999)]
+ metadata = _make_metadata([2], token_ids=token_ids, pooling_params=params)
+ out = pooler(hidden, metadata)
+ assert len(out) == 1
+ assert out[0].shape == (0, 2)
+
+ def test_chunked_prefill_propagates_none_for_unfinished(self):
+ pooler = self._make_step_pool(chunked=True)
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0]])
+ token_ids = [[10, 20, 30, 40]]
+ params = [PoolingParams(task="token_classify", step_tag_id=10)]
+ metadata = _make_metadata(
+ [4],
+ token_ids=token_ids,
+ pooling_params=params,
+ num_scheduled_tokens=[2],
+ seq_lens=[2],
+ )
+ out = pooler(hidden, metadata)
+ assert len(out) == 1
+ assert out[0] is None
+
+ def test_chunked_prefill_filters_when_finished(self):
+ pooler = self._make_step_pool(chunked=True)
+ hidden = torch.tensor([[1.0, 2.0], [3.0, 4.0], [5.0, 6.0], [7.0, 8.0]])
+ token_ids = [[10, 99, 10, 20]]
+ params = [PoolingParams(task="token_classify", step_tag_id=10)]
+ metadata = _make_metadata([4], token_ids=token_ids, pooling_params=params)
+ out = pooler(hidden, metadata)
+ assert len(out) == 1
+ expected = torch.tensor([[1.0, 2.0], [5.0, 6.0]])
+ assert torch.equal(out[0], expected)
+
+ def test_requires_token_ids_update(self):
+ pooler = self._make_step_pool()
+ update = pooler.get_pooling_updates("token_classify")
+ assert update.requires_token_ids is True
+
+
+# ---------------------------------------------------------------------------
+# get_tok_pooling_method factory
+# ---------------------------------------------------------------------------
+class TestGetTokPoolingMethod:
+ def test_all(self):
+ fake_config = _FakeVllmConfig(
+ scheduler_config=_FakeSchedulerConfig(
+ enable_chunked_prefill=False,
+ ),
+ )
+ with patch(
+ "vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
+ return_value=fake_config,
+ ):
+ assert isinstance(get_tok_pooling_method("ALL"), AllPool)
+
+ def test_step(self):
+ fake_config = _FakeVllmConfig(
+ scheduler_config=_FakeSchedulerConfig(
+ enable_chunked_prefill=False,
+ ),
+ )
+ with patch(
+ "vllm.model_executor.layers.pooler.tokwise.methods.get_current_vllm_config",
+ return_value=fake_config,
+ ):
+ assert isinstance(get_tok_pooling_method("STEP"), StepPool)
+
+ def test_unknown_raises(self):
+ with pytest.raises(NotImplementedError, match="UNKNOWN"):
+ get_tok_pooling_method("UNKNOWN")
diff --git a/asian_ai/performance/vLLM/tests/model_executor/layers/test_rocm_unquantized_gemm.py b/asian_ai/performance/vLLM/tests/model_executor/layers/test_rocm_unquantized_gemm.py
new file mode 100644
index 0000000000000000000000000000000000000000..07aec975407395a6eae2bbcbf91afedc650d9e0f
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/layers/test_rocm_unquantized_gemm.py
@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from unittest.mock import MagicMock
+
+import pytest
+import torch
+
+from vllm.platforms import current_platform
+
+if current_platform.is_cuda():
+ pytest.skip(
+ "ROCm skinny GEMM tests are not supported on CUDA.",
+ allow_module_level=True,
+ )
+
+from vllm.model_executor.layers import utils
+
+
+def test_rocm_unquantized_gemm_gfx1x_wvsplitk_path(monkeypatch):
+ x = torch.randn(1, 64, dtype=torch.float16)
+ weight = torch.randn(128, 64, dtype=torch.float16)
+
+ monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
+ monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
+ monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
+
+ wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
+ monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
+ llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
+ monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
+
+ out = utils.rocm_unquantized_gemm_impl(x, weight, None)
+ ref = torch.nn.functional.linear(x, weight, None)
+
+ wvsplitk_mock.assert_called_once()
+ llmm1_mock.assert_not_called()
+ assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
+
+
+def test_rocm_unquantized_gemm_gfx1x_n_gt_5_falls_back(monkeypatch):
+ # wvSplitK skinny GEMM handles n in [1, 5] (see PR #40687); n > 5 must
+ # fall back to torch.nn.functional.linear.
+ x = torch.randn(6, 64, dtype=torch.float16)
+ weight = torch.randn(128, 64, dtype=torch.float16)
+
+ monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
+ monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: True)
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: False)
+ monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
+
+ wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
+ monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
+ llmm1_mock = MagicMock(side_effect=lambda w, x_view, _: x_view @ w.t())
+ monkeypatch.setattr(utils.ops, "LLMM1", llmm1_mock)
+
+ out = utils.rocm_unquantized_gemm_impl(x, weight, None)
+ ref = torch.nn.functional.linear(x, weight, None)
+
+ wvsplitk_mock.assert_not_called()
+ llmm1_mock.assert_not_called()
+ assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
+
+
+def test_rocm_unquantized_gemm_gfx950_wvsplitkrc_path(monkeypatch):
+ x = torch.randn(16, 1024, dtype=torch.float16)
+ weight = torch.randn(256, 1024, dtype=torch.float16)
+
+ monkeypatch.setattr(utils, "use_aiter_triton_gemm", lambda *args: False)
+ monkeypatch.setattr(utils.envs, "VLLM_ROCM_USE_SKINNY_GEMM", True)
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx1x", lambda: False)
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx9", lambda: False)
+ monkeypatch.setattr("vllm.platforms.rocm.on_gfx950", lambda: True)
+ monkeypatch.setattr(utils, "num_compute_units", lambda: 120)
+
+ wvsplitkrc_mock = MagicMock(side_effect=lambda x_view, w, _, __: x_view @ w.t())
+ monkeypatch.setattr(utils.ops, "wvSplitKrc", wvsplitkrc_mock)
+ wvsplitk_mock = MagicMock(side_effect=lambda w, x_view, _, __: x_view @ w.t())
+ monkeypatch.setattr(utils.ops, "wvSplitK", wvsplitk_mock)
+
+ out = utils.rocm_unquantized_gemm_impl(x, weight, None)
+ ref = torch.nn.functional.linear(x, weight, None)
+
+ wvsplitkrc_mock.assert_called_once()
+ wvsplitk_mock.assert_not_called()
+ assert torch.allclose(out, ref, atol=1e-3, rtol=1e-3)
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/__init__.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/__init__.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/test_fastsafetensors_loader.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/test_fastsafetensors_loader.py
new file mode 100644
index 0000000000000000000000000000000000000000..cd4708ff79c5151a7f24f6ec2986a099c2074649
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/test_fastsafetensors_loader.py
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from vllm import SamplingParams
+from vllm.platforms import current_platform
+
+test_model = "openai-community/gpt2"
+
+prompts = [
+ "Hello, my name is",
+ "The president of the United States is",
+ "The capital of France is",
+ "The future of AI is",
+]
+# Create a sampling params object.
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
+
+
+@pytest.mark.skipif(
+ not current_platform.is_cuda_alike(),
+ reason="fastsafetensors requires NVIDIA/AMD GPUs",
+)
+def test_model_loader_download_files(vllm_runner):
+ with vllm_runner(test_model, load_format="fastsafetensors") as llm:
+ deserialized_outputs = llm.generate(prompts, sampling_params)
+ assert deserialized_outputs
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/test_weight_utils.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/test_weight_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..848a7854ba172bcf00567cb17342a05b84aecc46
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/fastsafetensors_loader/test_weight_utils.py
@@ -0,0 +1,49 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import glob
+import tempfile
+
+import huggingface_hub.constants
+import pytest
+import torch
+
+from vllm.model_executor.model_loader.weight_utils import (
+ download_weights_from_hf,
+ fastsafetensors_weights_iterator,
+ safetensors_weights_iterator,
+)
+from vllm.platforms import current_platform
+
+
+@pytest.mark.skipif(
+ not current_platform.is_cuda_alike(),
+ reason="fastsafetensors requires NVIDIA/AMD GPUs",
+)
+@pytest.mark.parametrize("queue_size", [0, 1])
+def test_fastsafetensors_model_loader(monkeypatch, queue_size):
+ monkeypatch.setenv("VLLM_FASTSAFETENSORS_QUEUE_SIZE", str(queue_size))
+ with tempfile.TemporaryDirectory() as tmpdir:
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
+ download_weights_from_hf(
+ "openai-community/gpt2", allow_patterns=["*.safetensors"], cache_dir=tmpdir
+ )
+ safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
+ assert len(safetensors) > 0
+
+ fastsafetensors_tensors = {}
+ hf_safetensors_tensors = {}
+
+ for name, tensor in fastsafetensors_weights_iterator(safetensors, True):
+ fastsafetensors_tensors[name] = tensor
+
+ for name, tensor in safetensors_weights_iterator(safetensors, True):
+ hf_safetensors_tensors[name] = tensor
+
+ assert len(fastsafetensors_tensors) == len(hf_safetensors_tensors)
+
+ for name, fastsafetensors_tensor in fastsafetensors_tensors.items():
+ fastsafetensors_tensor = fastsafetensors_tensor.to("cpu")
+ assert fastsafetensors_tensor.dtype == hf_safetensors_tensors[name].dtype
+ assert fastsafetensors_tensor.shape == hf_safetensors_tensors[name].shape
+ assert torch.all(fastsafetensors_tensor.eq(hf_safetensors_tensors[name]))
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/__init__.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/test_instanttensor_loader.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/test_instanttensor_loader.py
new file mode 100644
index 0000000000000000000000000000000000000000..d9c1b42b067a0d72c85c75c33eaa7683058fc158
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/test_instanttensor_loader.py
@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from vllm import SamplingParams
+from vllm.platforms import current_platform
+
+test_model = "openai-community/gpt2"
+
+prompts = [
+ "Hello, my name is",
+ "The president of the United States is",
+ "The capital of France is",
+ "The future of AI is",
+]
+# Create a sampling params object.
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
+
+
+@pytest.mark.skipif(
+ not current_platform.is_cuda(),
+ reason="InstantTensor requires NVIDIA GPUs",
+)
+def test_model_loader_download_files(vllm_runner):
+ with vllm_runner(test_model, load_format="instanttensor") as llm:
+ deserialized_outputs = llm.generate(prompts, sampling_params)
+ assert deserialized_outputs
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/test_weight_utils.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/test_weight_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..6f4351e1452141b5dc2d2be63500ac774756928b
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/instanttensor_loader/test_weight_utils.py
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import glob
+import tempfile
+
+import huggingface_hub.constants
+import pytest
+import torch
+
+from vllm.model_executor.model_loader.weight_utils import (
+ download_weights_from_hf,
+ instanttensor_weights_iterator,
+ safetensors_weights_iterator,
+)
+from vllm.platforms import current_platform
+
+
+@pytest.mark.skipif(
+ not current_platform.is_cuda(),
+ reason="InstantTensor requires NVIDIA GPUs",
+)
+def test_instanttensor_model_loader():
+ with tempfile.TemporaryDirectory() as tmpdir:
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
+ download_weights_from_hf(
+ "openai-community/gpt2", allow_patterns=["*.safetensors"], cache_dir=tmpdir
+ )
+ safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
+ assert len(safetensors) > 0
+
+ instanttensor_tensors = {}
+ hf_safetensors_tensors = {}
+
+ for name, tensor in instanttensor_weights_iterator(safetensors, True):
+ # Copy the tensor immediately as it is a reference to the internal
+ # buffer of instanttensor.
+ instanttensor_tensors[name] = tensor.to("cpu")
+
+ for name, tensor in safetensors_weights_iterator(safetensors, True):
+ hf_safetensors_tensors[name] = tensor
+
+ assert len(instanttensor_tensors) == len(hf_safetensors_tensors)
+
+ for name, instanttensor_tensor in instanttensor_tensors.items():
+ assert instanttensor_tensor.dtype == hf_safetensors_tensors[name].dtype
+ assert instanttensor_tensor.shape == hf_safetensors_tensors[name].shape
+ assert torch.all(instanttensor_tensor.eq(hf_safetensors_tensors[name]))
+
+
+if __name__ == "__main__":
+ test_instanttensor_model_loader()
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/__init__.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/conftest.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..a3c45d53ebada138259c0c4f85b0b4213f9c7a27
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/conftest.py
@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from vllm.utils.network_utils import get_distributed_init_method, get_ip, get_open_port
+from vllm.v1.executor import UniProcExecutor
+from vllm.v1.worker.worker_base import WorkerWrapperBase
+
+
+# This is a dummy executor for patching in test_runai_model_streamer_s3.py.
+# We cannot use vllm_runner fixture here, because it spawns worker process.
+# The worker process reimports the patched entities, and the patch is not applied.
+class RunaiDummyExecutor(UniProcExecutor):
+ def _init_executor(self) -> None:
+ distributed_init_method = get_distributed_init_method(get_ip(), get_open_port())
+
+ local_rank = 0
+ rank = 0
+ is_driver_worker = True
+
+ device_info = self.vllm_config.device_config.device.__str__().split(":")
+ if len(device_info) > 1:
+ local_rank = int(device_info[1])
+
+ worker_rpc_kwargs = dict(
+ vllm_config=self.vllm_config,
+ local_rank=local_rank,
+ rank=rank,
+ distributed_init_method=distributed_init_method,
+ is_driver_worker=is_driver_worker,
+ )
+
+ self.driver_worker = WorkerWrapperBase()
+
+ self.collective_rpc("init_worker", args=([worker_rpc_kwargs],))
+ self.collective_rpc("init_device")
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_loader.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_loader.py
new file mode 100644
index 0000000000000000000000000000000000000000..d514ef8b4c652741cf41cb646b90fd96e6672362
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_loader.py
@@ -0,0 +1,124 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import os
+import types
+from unittest.mock import patch
+
+import pytest
+
+from vllm import SamplingParams
+from vllm.config.load import LoadConfig
+from vllm.model_executor.model_loader import get_model_loader
+from vllm.model_executor.model_loader import runai_streamer_loader as rsl
+
+load_format = "runai_streamer"
+test_model = "openai-community/gpt2"
+# TODO(amacaskill): Replace with a GKE owned GCS bucket.
+test_gcs_model = "gs://vertex-model-garden-public-us/codegemma/codegemma-2b/"
+
+prompts = [
+ "Hello, my name is",
+ "The president of the United States is",
+ "The capital of France is",
+ "The future of AI is",
+]
+# Create a sampling params object.
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
+
+
+def get_runai_model_loader():
+ load_config = LoadConfig(load_format=load_format)
+ return get_model_loader(load_config)
+
+
+def test_get_model_loader_with_runai_flag():
+ model_loader = get_runai_model_loader()
+ assert model_loader.__class__.__name__ == "RunaiModelStreamerLoader"
+
+
+def test_runai_model_loader_download_files(vllm_runner):
+ with vllm_runner(test_model, load_format=load_format) as llm:
+ deserialized_outputs = llm.generate(prompts, sampling_params)
+ assert deserialized_outputs
+
+
+@pytest.mark.skip(
+ reason="Temporarily disabled due to GCS access issues. "
+ "TODO: Re-enable this test once the underlying issue is resolved."
+)
+def test_runai_model_loader_download_files_gcs(
+ vllm_runner, monkeypatch: pytest.MonkeyPatch
+):
+ monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "fake-project")
+ monkeypatch.setenv("RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS", "true")
+ monkeypatch.setenv(
+ "CLOUD_STORAGE_EMULATOR_ENDPOINT", "https://storage.googleapis.com"
+ )
+ with vllm_runner(test_gcs_model, load_format=load_format) as llm:
+ deserialized_outputs = llm.generate(prompts, sampling_params)
+ assert deserialized_outputs
+
+
+def test_runai_passes_revision_by_name():
+ # revision must reach download_safetensors_index_file_from_hf as the
+ # ``revision`` keyword, not the positional ``subfolder`` slot.
+ fake_self = types.SimpleNamespace(
+ load_config=types.SimpleNamespace(download_dir="/cache", ignore_patterns=[])
+ )
+ with (
+ patch.object(rsl, "is_runai_obj_uri", return_value=False),
+ patch.object(rsl, "download_weights_from_hf", return_value="/folder"),
+ patch.object(
+ rsl, "list_safetensors", return_value=["/folder/model.safetensors"]
+ ),
+ patch.object(rsl, "download_safetensors_index_file_from_hf") as mock_idx,
+ ):
+ rsl.RunaiModelStreamerLoader._prepare_weights(fake_self, "org/model", "myrev")
+
+ mock_idx.assert_called_once()
+ assert mock_idx.call_args.kwargs.get("revision") == "myrev"
+ assert "myrev" not in mock_idx.call_args.args
+
+
+def _runai_loader(extra):
+ return rsl.RunaiModelStreamerLoader(
+ LoadConfig(load_format="runai_streamer", model_loader_extra_config=extra)
+ )
+
+
+@pytest.mark.parametrize(
+ "extra, match",
+ [
+ ({"typo_key": 1}, "Unexpected extra config"),
+ ({"distributed": "yes"}, "distributed must be a bool"),
+ ({"concurrency": "16"}, "concurrency must be a positive integer"),
+ ({"concurrency": -1}, "concurrency must be a positive integer"),
+ ],
+)
+def test_runai_rejects_invalid_extra_config(extra, match):
+ # The loader used to silently drop unknown keys / wrong types / negatives.
+ with pytest.raises(ValueError, match=match):
+ _runai_loader(extra)
+
+
+def test_runai_accepts_valid_extra_config():
+ with patch.dict(os.environ, {}, clear=False):
+ os.environ.pop("RUNAI_STREAMER_CONCURRENCY", None)
+ os.environ.pop("RUNAI_STREAMER_MEMORY_LIMIT", None)
+ loader = _runai_loader(
+ {"distributed": True, "concurrency": 16, "memory_limit": 1024}
+ )
+ assert loader._is_distributed is True
+ assert os.environ["RUNAI_STREAMER_CONCURRENCY"] == "16"
+ assert os.environ["RUNAI_STREAMER_MEMORY_LIMIT"] == "1024"
+
+
+def test_runai_invalid_extra_config_leaves_environ_untouched():
+ # A later invalid key must not leave an earlier valid key applied to
+ # os.environ (all values are validated before any global mutation).
+ with patch.dict(os.environ, {}, clear=False):
+ os.environ.pop("RUNAI_STREAMER_CONCURRENCY", None)
+ with pytest.raises(ValueError, match="memory_limit must be a positive integer"):
+ _runai_loader({"concurrency": 16, "memory_limit": -5})
+ assert "RUNAI_STREAMER_CONCURRENCY" not in os.environ
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py
new file mode 100644
index 0000000000000000000000000000000000000000..6abb10560e8e46b170f4757bc1bfc70bb7d8035a
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_model_streamer_s3.py
@@ -0,0 +1,52 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from pathlib import Path
+
+from huggingface_hub import snapshot_download
+from runai_model_streamer.safetensors_streamer.streamer_mock import StreamerPatcher
+
+from vllm.engine.arg_utils import EngineArgs
+
+from .conftest import RunaiDummyExecutor
+
+load_format = "runai_streamer"
+test_model = "openai-community/gpt2"
+
+
+def test_runai_model_loader_download_files_s3_mocked_with_patch(
+ vllm_runner,
+ tmp_path: Path,
+ monkeypatch,
+):
+ patcher = StreamerPatcher(str(tmp_path))
+
+ test_mock_s3_model = "s3://my-mock-bucket/gpt2/"
+
+ # Download model from HF
+ mock_model_dir = f"{tmp_path}/gpt2"
+ snapshot_download(repo_id=test_model, local_dir=mock_model_dir)
+
+ monkeypatch.setattr(
+ "vllm.transformers_utils.runai_utils.runai_list_safetensors",
+ patcher.shim_list_safetensors,
+ )
+ monkeypatch.setattr(
+ "vllm.transformers_utils.runai_utils.runai_pull_files",
+ patcher.shim_pull_files,
+ )
+ monkeypatch.setattr(
+ "vllm.model_executor.model_loader.weight_utils.SafetensorsStreamer",
+ patcher.create_mock_streamer,
+ )
+
+ engine_args = EngineArgs(
+ model=test_mock_s3_model,
+ load_format=load_format,
+ tensor_parallel_size=1,
+ )
+
+ vllm_config = engine_args.create_engine_config()
+
+ executor = RunaiDummyExecutor(vllm_config)
+ executor.driver_worker.load_model()
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_utils.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..04f0c8870438b456d2bc315f84253f39c03518b0
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_runai_utils.py
@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import glob
+import hashlib
+import os
+import tempfile
+
+import huggingface_hub.constants
+
+from vllm.model_executor.model_loader.weight_utils import download_weights_from_hf
+from vllm.transformers_utils.runai_utils import (
+ ObjectStorageModel,
+ is_runai_obj_uri,
+ list_safetensors,
+)
+
+
+def test_is_runai_obj_uri():
+ assert is_runai_obj_uri("gs://some-gcs-bucket/path")
+ assert is_runai_obj_uri("s3://some-s3-bucket/path")
+ assert is_runai_obj_uri("az://some-azure-container/path")
+ assert not is_runai_obj_uri("nfs://some-nfs-path")
+
+
+def test_runai_list_safetensors_local():
+ with tempfile.TemporaryDirectory() as tmpdir:
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
+ download_weights_from_hf(
+ "openai-community/gpt2",
+ allow_patterns=["*.safetensors", "*.json"],
+ cache_dir=tmpdir,
+ )
+ safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
+ assert len(safetensors) > 0
+ parentdir = [os.path.dirname(safetensor) for safetensor in safetensors][0]
+ files = list_safetensors(parentdir)
+ assert len(safetensors) == len(files)
+
+
+def test_runai_pull_files_gcs(monkeypatch):
+ monkeypatch.setenv("RUNAI_STREAMER_GCS_USE_ANONYMOUS_CREDENTIALS", "true")
+ # Bypass default project lookup by setting GOOGLE_CLOUD_PROJECT
+ monkeypatch.setenv("GOOGLE_CLOUD_PROJECT", "fake-project")
+ filename = "LT08_L1GT_074061_20130309_20170505_01_T2_MTL.txt"
+ gcs_bucket = "gs://gcp-public-data-landsat/LT08/01/074/061/LT08_L1GT_074061_20130309_20170505_01_T2/"
+ gcs_url = f"{gcs_bucket}/{filename}"
+ model = ObjectStorageModel(gcs_url)
+ model.pull_files(gcs_bucket, allow_pattern=[f"*{filename}"])
+ # To re-generate / change URLs:
+ # gsutil ls -L gs:// | grep "Hash (md5)" | tr -d ' ' \
+ # | cut -d":" -f2 | base64 -d | xxd -p
+ expected_checksum = "f60dea775da1392434275b311b31a431"
+ hasher = hashlib.new("md5")
+ with open(os.path.join(model.dir, filename), "rb") as f:
+ # Read the file in chunks to handle large files efficiently
+ for chunk in iter(lambda: f.read(4096), b""):
+ hasher.update(chunk)
+ actual_checksum = hasher.hexdigest()
+ assert actual_checksum == expected_checksum
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_weight_utils.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_weight_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..4deae649d8070cae4314fd5fa35e07eee6f8b0db
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/runai_streamer_loader/test_weight_utils.py
@@ -0,0 +1,66 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import glob
+import tempfile
+
+import huggingface_hub.constants
+import torch
+from safetensors.torch import save_file
+
+from vllm.model_executor.model_loader.weight_utils import (
+ download_weights_from_hf,
+ runai_safetensors_weights_iterator,
+ safetensors_weights_iterator,
+)
+
+
+def test_runai_safetensors_weights_iterator_clones_reused_buffers(
+ tmp_path, monkeypatch
+):
+ monkeypatch.setenv("RUNAI_STREAMER_MEMORY_LIMIT", "0")
+ weights_file = tmp_path / "model.safetensors"
+ expected_tensors = {
+ "first": torch.tensor([1.0, 2.0]),
+ "second": torch.tensor([3.0, 4.0]),
+ }
+ save_file(expected_tensors, weights_file)
+
+ actual_tensors = dict(
+ runai_safetensors_weights_iterator([str(weights_file)], False)
+ )
+
+ assert actual_tensors.keys() == expected_tensors.keys()
+ assert actual_tensors["first"].data_ptr() != actual_tensors["second"].data_ptr()
+ for name, expected_tensor in expected_tensors.items():
+ assert torch.equal(actual_tensors[name], expected_tensor)
+
+
+def test_runai_model_loader():
+ with tempfile.TemporaryDirectory() as tmpdir:
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
+ download_weights_from_hf(
+ "openai-community/gpt2", allow_patterns=["*.safetensors"], cache_dir=tmpdir
+ )
+ safetensors = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
+ assert len(safetensors) > 0
+
+ runai_model_streamer_tensors = {}
+ hf_safetensors_tensors = {}
+
+ for name, tensor in runai_safetensors_weights_iterator(safetensors, True):
+ runai_model_streamer_tensors[name] = tensor
+
+ for name, tensor in safetensors_weights_iterator(safetensors, True):
+ hf_safetensors_tensors[name] = tensor
+
+ assert len(runai_model_streamer_tensors) == len(hf_safetensors_tensors)
+
+ for name, runai_tensor in runai_model_streamer_tensors.items():
+ assert runai_tensor.dtype == hf_safetensors_tensors[name].dtype
+ assert runai_tensor.shape == hf_safetensors_tensors[name].shape
+ assert torch.all(runai_tensor.eq(hf_safetensors_tensors[name]))
+
+
+if __name__ == "__main__":
+ test_runai_model_loader()
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/__init__.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/conftest.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..92008e23093d0855b883732c32ad549d3da99a1e
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/conftest.py
@@ -0,0 +1,92 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from collections.abc import Callable
+
+import pytest
+
+from vllm import LLM, EngineArgs
+from vllm.distributed import cleanup_dist_env_and_memory
+from vllm.model_executor.model_loader import tensorizer as tensorizer_mod
+from vllm.model_executor.model_loader.tensorizer import TensorizerConfig
+from vllm.utils.network_utils import get_distributed_init_method, get_ip, get_open_port
+from vllm.v1.executor import UniProcExecutor
+from vllm.v1.worker.worker_base import WorkerWrapperBase
+
+MODEL_REF = "facebook/opt-125m"
+
+
+@pytest.fixture()
+def model_ref():
+ return MODEL_REF
+
+
+@pytest.fixture(autouse=True)
+def allow_insecure_serialization(monkeypatch):
+ monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+
+
+@pytest.fixture(autouse=True)
+def cleanup():
+ cleanup_dist_env_and_memory(shutdown_ray=True)
+
+
+@pytest.fixture()
+def just_serialize_model_tensors(model_ref, monkeypatch, tmp_path):
+ def noop(*args, **kwargs):
+ return None
+
+ args = EngineArgs(model=model_ref)
+ tc = TensorizerConfig(tensorizer_uri=f"{tmp_path}/model.tensors")
+
+ monkeypatch.setattr(tensorizer_mod, "serialize_extra_artifacts", noop)
+
+ tensorizer_mod.tensorize_vllm_model(args, tc)
+ yield tmp_path
+
+
+@pytest.fixture(autouse=True)
+def tensorizer_config():
+ config = TensorizerConfig(tensorizer_uri="vllm")
+ return config
+
+
+@pytest.fixture()
+def model_path(model_ref, tmp_path):
+ yield tmp_path / model_ref / "model.tensors"
+
+
+def assert_from_collective_rpc(engine: LLM, closure: Callable, closure_kwargs: dict):
+ res = engine.collective_rpc(method=closure, kwargs=closure_kwargs)
+ return all(res)
+
+
+# This is an object pulled from tests/v1/engine/test_engine_core.py
+# Modified to strip the `load_model` method from its `_init_executor`
+# method. It's purely used as a dummy utility to run methods that test
+# Tensorizer functionality
+class DummyExecutor(UniProcExecutor):
+ def _init_executor(self) -> None:
+ """Initialize the worker and load the model."""
+ self.driver_worker = WorkerWrapperBase(rpc_rank=0)
+ distributed_init_method = get_distributed_init_method(get_ip(), get_open_port())
+ local_rank = 0
+ # set local rank as the device index if specified
+ device_info = self.vllm_config.device_config.device.__str__().split(":")
+ if len(device_info) > 1:
+ local_rank = int(device_info[1])
+ rank = 0
+ is_driver_worker = True
+ kwargs = dict(
+ vllm_config=self.vllm_config,
+ local_rank=local_rank,
+ rank=rank,
+ distributed_init_method=distributed_init_method,
+ is_driver_worker=is_driver_worker,
+ )
+ self.mm_receiver_cache = None
+ self.collective_rpc("init_worker", args=([kwargs],))
+ self.collective_rpc("init_device")
+
+ def shutdown(self):
+ if hasattr(self, "thread_pool"):
+ self.thread_pool.shutdown(wait=False)
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py
new file mode 100644
index 0000000000000000000000000000000000000000..47c945977b30466a510a6c8f0a32574346db89c9
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/tensorizer_loader/test_tensorizer.py
@@ -0,0 +1,560 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import asyncio
+import gc
+import json
+import os
+import pathlib
+import subprocess
+import sys
+from typing import Any
+
+import pytest
+import torch
+
+import vllm.model_executor.model_loader.tensorizer
+from tests.utils import VLLM_PATH, RemoteOpenAIServer
+from vllm import LLM, SamplingParams
+from vllm.engine.arg_utils import EngineArgs
+from vllm.model_executor.model_loader.tensorizer import (
+ TensorizerConfig,
+ TensorSerializer,
+ is_vllm_tensorized,
+ open_stream,
+ tensorize_vllm_model,
+)
+from vllm.model_executor.model_loader.tensorizer_loader import (
+ BLACKLISTED_TENSORIZER_ARGS,
+)
+from vllm.utils.import_utils import PlaceholderModule
+
+from .conftest import DummyExecutor, assert_from_collective_rpc
+
+try:
+ import tensorizer
+ from tensorizer import EncryptionParams
+except ImportError:
+ tensorizer = PlaceholderModule("tensorizer") # type: ignore[assignment]
+ EncryptionParams = tensorizer.placeholder_attr("EncryptionParams")
+
+
+class TensorizerCaughtError(Exception):
+ pass
+
+
+EXAMPLES_PATH = VLLM_PATH / "examples"
+
+pytest_plugins = ("pytest_asyncio",)
+
+prompts = [
+ "Hello, my name is",
+ "The president of the United States is",
+ "The capital of France is",
+ "The future of AI is",
+]
+# Create a sampling params object.
+sampling_params = SamplingParams(temperature=0.8, top_p=0.95, seed=0)
+
+
+def patch_init_and_catch_error(self, obj, method_name, expected_error: type[Exception]):
+ original = getattr(obj, method_name, None)
+ if original is None:
+ raise ValueError("Method '{}' not found.".format(method_name))
+
+ def wrapper(*args, **kwargs):
+ try:
+ return original(*args, **kwargs)
+ except expected_error as err:
+ raise TensorizerCaughtError from err
+
+ setattr(obj, method_name, wrapper)
+
+ self.load_model()
+
+
+def assert_specific_tensorizer_error_is_raised(
+ executor,
+ obj: Any,
+ method_name: str,
+ expected_error: type[Exception],
+):
+ with pytest.raises(TensorizerCaughtError):
+ executor.collective_rpc(
+ patch_init_and_catch_error,
+ args=(
+ obj,
+ method_name,
+ expected_error,
+ ),
+ )
+
+
+def is_curl_installed():
+ try:
+ subprocess.check_call(["curl", "--version"])
+ return True
+ except (subprocess.CalledProcessError, FileNotFoundError):
+ return False
+
+
+def write_keyfile(keyfile_path: str):
+ encryption_params = EncryptionParams.random()
+ pathlib.Path(keyfile_path).parent.mkdir(parents=True, exist_ok=True)
+ with open(keyfile_path, "wb") as f:
+ f.write(encryption_params.key)
+
+
+@pytest.mark.skipif(not is_curl_installed(), reason="cURL is not installed")
+def test_deserialized_encrypted_vllm_model_has_same_outputs(
+ model_ref, vllm_runner, tmp_path, model_path
+):
+ args = EngineArgs(model=model_ref)
+ with vllm_runner(model_ref) as vllm_model:
+ key_path = tmp_path / model_ref / "model.key"
+ write_keyfile(key_path)
+
+ outputs = vllm_model.generate(prompts, sampling_params)
+
+ config_for_serializing = TensorizerConfig(
+ tensorizer_uri=str(model_path), encryption_keyfile=str(key_path)
+ )
+
+ tensorize_vllm_model(args, config_for_serializing)
+
+ config_for_deserializing = TensorizerConfig(
+ tensorizer_uri=str(model_path), encryption_keyfile=str(key_path)
+ )
+
+ with vllm_runner(
+ model_ref,
+ load_format="tensorizer",
+ model_loader_extra_config=config_for_deserializing,
+ ) as loaded_vllm_model: # noqa: E501
+ deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
+ # noqa: E501
+
+ assert outputs == deserialized_outputs
+
+
+def test_deserialized_hf_model_has_same_outputs(
+ hf_runner, vllm_runner, tmp_path, model_ref, model_path
+):
+ with hf_runner(model_ref) as hf_model:
+ max_tokens = 50
+ outputs = hf_model.generate_greedy(prompts, max_tokens=max_tokens)
+ with open_stream(model_path, "wb+") as stream:
+ serializer = TensorSerializer(stream)
+ serializer.write_module(hf_model.model)
+
+ with vllm_runner(
+ model_ref,
+ load_format="tensorizer",
+ model_loader_extra_config=TensorizerConfig(
+ tensorizer_uri=str(model_path),
+ num_readers=1,
+ ),
+ ) as loaded_hf_model:
+ deserialized_outputs = loaded_hf_model.generate_greedy(
+ prompts, max_tokens=max_tokens
+ )
+
+ assert outputs == deserialized_outputs
+
+
+def test_load_without_tensorizer_load_format(vllm_runner, capfd, model_ref):
+ model = None
+ try:
+ model = vllm_runner(
+ model_ref, model_loader_extra_config=TensorizerConfig(tensorizer_uri="test")
+ )
+ pytest.fail("Expected RuntimeError for extra config keys")
+ except RuntimeError:
+ out, err = capfd.readouterr()
+ combined_output = out + err
+ assert (
+ "ValueError: Unexpected extra config keys for load format auto"
+ ) in combined_output
+ finally:
+ del model
+ gc.collect()
+ torch.accelerator.empty_cache()
+
+
+def test_raise_value_error_on_invalid_load_format(vllm_runner, capfd, model_ref):
+ model = None
+ try:
+ model = vllm_runner(
+ model_ref,
+ load_format="safetensors",
+ model_loader_extra_config=TensorizerConfig(tensorizer_uri="test"),
+ )
+ pytest.fail("Expected RuntimeError for extra config keys")
+ except RuntimeError:
+ out, err = capfd.readouterr()
+
+ combined_output = out + err
+ assert (
+ "ValueError: Unexpected extra config keys for load format safetensors"
+ ) in combined_output
+ finally:
+ del model
+ gc.collect()
+ torch.accelerator.empty_cache()
+
+
+@pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Requires 2 GPUs")
+def test_tensorizer_with_tp_path_without_template(vllm_runner, capfd):
+ try:
+ model_ref = "EleutherAI/pythia-1.4b"
+ tensorized_path = f"s3://tensorized/{model_ref}/fp16/model.tensors"
+
+ vllm_runner(
+ model_ref,
+ load_format="tensorizer",
+ model_loader_extra_config=TensorizerConfig(
+ tensorizer_uri=tensorized_path,
+ num_readers=1,
+ s3_endpoint="object.ord1.coreweave.com",
+ ),
+ tensor_parallel_size=2,
+ disable_custom_all_reduce=True,
+ )
+ except RuntimeError:
+ out, err = capfd.readouterr()
+ combined_output = out + err
+ assert (
+ "ValueError: For a sharded model, tensorizer_uri "
+ "should include a string format template like '%04d' "
+ "to be formatted with the rank "
+ "of the shard"
+ ) in combined_output
+
+
+@pytest.mark.skipif(torch.accelerator.device_count() < 2, reason="Requires 2 GPUs")
+def test_deserialized_encrypted_vllm_model_with_tp_has_same_outputs(
+ vllm_runner, tmp_path
+):
+ model_ref = "EleutherAI/pythia-1.4b"
+ # record outputs from un-sharded un-tensorized model
+ with vllm_runner(
+ model_ref,
+ disable_custom_all_reduce=True,
+ enforce_eager=True,
+ ) as base_model:
+ outputs = base_model.generate(prompts, sampling_params)
+
+ # load model with two shards and serialize with encryption
+ model_path = str(tmp_path / model_ref / "model-%02d.tensors")
+ key_path = tmp_path / (model_ref + ".key")
+
+ tensorizer_config = TensorizerConfig(
+ tensorizer_uri=model_path,
+ encryption_keyfile=str(key_path),
+ )
+
+ tensorize_vllm_model(
+ engine_args=EngineArgs(
+ model=model_ref,
+ tensor_parallel_size=2,
+ disable_custom_all_reduce=True,
+ enforce_eager=True,
+ ),
+ tensorizer_config=tensorizer_config,
+ )
+ assert os.path.isfile(model_path % 0), "Serialization subprocess failed"
+ assert os.path.isfile(model_path % 1), "Serialization subprocess failed"
+
+ with vllm_runner(
+ model_ref,
+ tensor_parallel_size=2,
+ load_format="tensorizer",
+ disable_custom_all_reduce=True,
+ enforce_eager=True,
+ model_loader_extra_config=tensorizer_config,
+ ) as loaded_vllm_model:
+ deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
+
+ assert outputs == deserialized_outputs
+
+
+@pytest.mark.flaky(reruns=3)
+def test_vllm_tensorized_model_has_same_outputs(
+ model_ref, vllm_runner, tmp_path, model_path
+):
+ gc.collect()
+ torch.accelerator.empty_cache()
+ config = TensorizerConfig(tensorizer_uri=str(model_path))
+ args = EngineArgs(model=model_ref)
+
+ with vllm_runner(model_ref) as vllm_model:
+ outputs = vllm_model.generate(prompts, sampling_params)
+
+ tensorize_vllm_model(args, config)
+ assert is_vllm_tensorized(config)
+
+ with vllm_runner(
+ model_ref, load_format="tensorizer", model_loader_extra_config=config
+ ) as loaded_vllm_model:
+ deserialized_outputs = loaded_vllm_model.generate(prompts, sampling_params)
+ # noqa: E501
+
+ assert outputs == deserialized_outputs
+
+
+def test_load_with_just_model_tensors(just_serialize_model_tensors, model_ref):
+ # For backwards compatibility, ensure Tensorizer can be still be loaded
+ # for inference by passing the model reference name, not a local/S3 dir,
+ # and the location of the model tensors
+
+ model_dir = just_serialize_model_tensors
+
+ extra_config = {"tensorizer_uri": f"{model_dir}/model.tensors"}
+
+ ## Start OpenAI API server
+ args = [
+ "--load-format",
+ "tensorizer",
+ "--model-loader-extra-config",
+ json.dumps(extra_config),
+ ]
+
+ with RemoteOpenAIServer(model_ref, args):
+ # This test only concerns itself with being able to load the model
+ # and successfully initialize the server
+ pass
+
+
+def test_assert_serialization_kwargs_passed_to_tensor_serializer(tmp_path):
+ serialization_params = {
+ "limit_cpu_concurrency": 2,
+ }
+ model_ref = "facebook/opt-125m"
+ model_path = tmp_path / (model_ref + ".tensors")
+ config = TensorizerConfig(
+ tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
+ )
+ llm = LLM(
+ model=model_ref,
+ )
+
+ def serialization_test(self, *args, **kwargs):
+ # This is performed in the ephemeral worker process, so monkey-patching
+ # will actually work, and cleanup is guaranteed so don't
+ # need to reset things
+
+ original_dict = serialization_params
+ to_compare = {}
+
+ original = tensorizer.serialization.TensorSerializer.__init__
+
+ def tensorizer_serializer_wrapper(self, *args, **kwargs):
+ nonlocal to_compare
+ to_compare = kwargs.copy()
+ return original(self, *args, **kwargs)
+
+ tensorizer.serialization.TensorSerializer.__init__ = (
+ tensorizer_serializer_wrapper
+ )
+
+ tensorizer_config = TensorizerConfig(**kwargs["tensorizer_config"])
+ self.save_tensorized_model(
+ tensorizer_config=tensorizer_config,
+ )
+ return to_compare | original_dict == to_compare
+
+ kwargs = {"tensorizer_config": config.to_serializable()}
+
+ assert assert_from_collective_rpc(llm, serialization_test, kwargs)
+
+
+def test_assert_deserialization_kwargs_passed_to_tensor_deserializer(tmp_path, capfd):
+ deserialization_kwargs = {
+ "num_readers": "bar", # illegal value
+ }
+
+ serialization_params = {
+ "limit_cpu_concurrency": 2,
+ }
+
+ model_ref = "facebook/opt-125m"
+ model_path = tmp_path / (model_ref + ".tensors")
+ config = TensorizerConfig(
+ tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
+ )
+
+ args = EngineArgs(model=model_ref)
+ tensorize_vllm_model(args, config)
+
+ loader_tc = TensorizerConfig(
+ tensorizer_uri=str(model_path),
+ deserialization_kwargs=deserialization_kwargs,
+ )
+
+ engine_args = EngineArgs(
+ model="facebook/opt-125m",
+ load_format="tensorizer",
+ model_loader_extra_config=loader_tc.to_serializable(),
+ )
+
+ vllm_config = engine_args.create_engine_config()
+ executor = DummyExecutor(vllm_config)
+
+ assert_specific_tensorizer_error_is_raised(
+ executor,
+ tensorizer.serialization.TensorDeserializer,
+ "__init__",
+ TypeError,
+ )
+
+
+def test_assert_stream_kwargs_passed_to_tensor_deserializer(tmp_path, capfd):
+ deserialization_kwargs = {
+ "num_readers": 1,
+ }
+
+ serialization_params = {
+ "limit_cpu_concurrency": 2,
+ }
+
+ model_ref = "facebook/opt-125m"
+ model_path = tmp_path / (model_ref + ".tensors")
+ config = TensorizerConfig(
+ tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
+ )
+
+ args = EngineArgs(model=model_ref)
+ tensorize_vllm_model(args, config)
+
+ stream_kwargs = {"mode": "foo"}
+
+ loader_tc = TensorizerConfig(
+ tensorizer_uri=str(model_path),
+ deserialization_kwargs=deserialization_kwargs,
+ stream_kwargs=stream_kwargs,
+ )
+
+ engine_args = EngineArgs(
+ model="facebook/opt-125m",
+ load_format="tensorizer",
+ model_loader_extra_config=loader_tc.to_serializable(),
+ )
+
+ vllm_config = engine_args.create_engine_config()
+ executor = DummyExecutor(vllm_config)
+
+ assert_specific_tensorizer_error_is_raised(
+ executor,
+ vllm.model_executor.model_loader.tensorizer,
+ "open_stream",
+ ValueError,
+ )
+
+
+@pytest.mark.asyncio
+async def test_serialize_and_serve_entrypoints(tmp_path):
+ model_ref = "facebook/opt-125m"
+
+ suffix = "test"
+ try:
+ result = subprocess.run(
+ [
+ sys.executable,
+ f"{VLLM_PATH}/examples/features/tensorize_vllm_model.py",
+ "--model",
+ model_ref,
+ "serialize",
+ "--serialized-directory",
+ str(tmp_path),
+ "--suffix",
+ suffix,
+ "--serialization-kwargs",
+ '{"limit_cpu_concurrency": 4}',
+ ],
+ check=True,
+ capture_output=True,
+ text=True,
+ )
+ except subprocess.CalledProcessError as e:
+ print("Tensorizing failed.")
+ print("STDOUT:\n", e.stdout)
+ print("STDERR:\n", e.stderr)
+ raise
+
+ assert "Successfully serialized" in result.stdout
+
+ # Next, try to serve with vllm serve
+ model_uri = tmp_path / "vllm" / model_ref / suffix / "model.tensors"
+
+ model_loader_extra_config = {
+ "tensorizer_uri": str(model_uri),
+ "stream_kwargs": {
+ "force_http": False,
+ },
+ "deserialization_kwargs": {
+ "verify_hash": True,
+ "num_readers": 8,
+ },
+ }
+
+ cmd = [
+ "-m",
+ "vllm.entrypoints.cli.main",
+ "serve",
+ "--host",
+ "localhost",
+ "--load-format",
+ "tensorizer",
+ model_ref,
+ "--model-loader-extra-config",
+ json.dumps(model_loader_extra_config, indent=2),
+ ]
+
+ proc = await asyncio.create_subprocess_exec(
+ sys.executable,
+ *cmd,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.STDOUT,
+ )
+
+ assert proc.stdout is not None
+ fut = proc.stdout.readuntil(b"Application startup complete.")
+
+ try:
+ await asyncio.wait_for(fut, 180)
+ except asyncio.TimeoutError:
+ pytest.fail("Server did not start successfully")
+ finally:
+ proc.terminate()
+ await proc.communicate()
+
+
+@pytest.mark.parametrize("illegal_value", BLACKLISTED_TENSORIZER_ARGS)
+def test_blacklisted_parameter_for_loading(tmp_path, vllm_runner, capfd, illegal_value):
+ serialization_params = {
+ "limit_cpu_concurrency": 2,
+ }
+
+ model_ref = "facebook/opt-125m"
+ model_path = tmp_path / (model_ref + ".tensors")
+ config = TensorizerConfig(
+ tensorizer_uri=str(model_path), serialization_kwargs=serialization_params
+ )
+
+ args = EngineArgs(model=model_ref)
+ tensorize_vllm_model(args, config)
+
+ loader_tc = {"tensorizer_uri": str(model_path), illegal_value: "foo"}
+
+ try:
+ vllm_runner(
+ model_ref,
+ load_format="tensorizer",
+ model_loader_extra_config=loader_tc,
+ )
+ except RuntimeError:
+ out, err = capfd.readouterr()
+ combined_output = out + err
+ assert (
+ f"ValueError: {illegal_value} is not an allowed Tensorizer argument."
+ ) in combined_output
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_ep_weight_filter.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_ep_weight_filter.py
new file mode 100644
index 0000000000000000000000000000000000000000..4b3aebd28409a827b26bcd417e29f9186ae7567e
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_ep_weight_filter.py
@@ -0,0 +1,361 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for EP weight filtering during model loading."""
+
+import glob
+import tempfile
+
+import huggingface_hub.constants
+import pytest
+import torch
+
+from vllm.model_executor.model_loader.ep_weight_filter import (
+ compute_local_expert_ids,
+ parse_expert_id,
+ should_skip_weight,
+)
+from vllm.model_executor.model_loader.weight_utils import (
+ safetensors_weights_iterator,
+)
+
+# ---------------------------------------------------------------------------
+# Unit tests for parse_expert_id
+# ---------------------------------------------------------------------------
+
+
+class TestParseExpertId:
+ def test_routed_expert(self):
+ name = "model.layers.0.mlp.experts.42.gate_proj.weight"
+ assert parse_expert_id(name) == 42
+
+ def test_large_expert_id(self):
+ name = "model.layers.60.mlp.experts.383.down_proj.weight"
+ assert parse_expert_id(name) == 383
+
+ def test_shared_expert(self):
+ # Shared experts use a different naming convention in most models
+ name = "model.layers.0.mlp.shared_experts.gate_proj.weight"
+ assert parse_expert_id(name) is None
+
+ def test_attention_weight(self):
+ name = "model.layers.0.self_attn.q_proj.weight"
+ assert parse_expert_id(name) is None
+
+ def test_embedding(self):
+ name = "model.embed_tokens.weight"
+ assert parse_expert_id(name) is None
+
+ def test_layernorm(self):
+ name = "model.layers.0.input_layernorm.weight"
+ assert parse_expert_id(name) is None
+
+ def test_fused_3d_expert(self):
+ # 3D fused-expert tensors (e.g. gpt-oss) have no numeric expert id.
+ # They must NOT be filtered — slicing happens later in weight_loader.
+ name = "model.layers.0.mlp.experts.gate_proj.weight"
+ assert parse_expert_id(name) is None
+
+ def test_fused_3d_expert_down_proj(self):
+ name = "model.layers.10.mlp.experts.down_proj.weight"
+ assert parse_expert_id(name) is None
+
+ def test_expert_scale(self):
+ # NVFP4 quantized models have scale tensors for experts
+ name = "model.layers.5.mlp.experts.100.gate_proj.weight_scale"
+ assert parse_expert_id(name) == 100
+
+ def test_expert_zero_id(self):
+ name = "model.layers.0.mlp.experts.0.up_proj.weight"
+ assert parse_expert_id(name) == 0
+
+
+# ---------------------------------------------------------------------------
+# Unit tests for compute_local_expert_ids
+# ---------------------------------------------------------------------------
+
+
+class TestComputeLocalExpertIds:
+ def test_ep_disabled(self):
+ assert compute_local_expert_ids(64, ep_size=1, ep_rank=0) is None
+
+ def test_even_split(self):
+ # 64 experts, EP=8 → 8 per rank
+ ids = compute_local_expert_ids(64, ep_size=8, ep_rank=0)
+ assert ids == set(range(0, 8))
+
+ ids = compute_local_expert_ids(64, ep_size=8, ep_rank=7)
+ assert ids == set(range(56, 64))
+
+ def test_uneven_split(self):
+ # 10 experts, EP=3 → ranks get 4, 3, 3
+ ids_0 = compute_local_expert_ids(10, ep_size=3, ep_rank=0)
+ ids_1 = compute_local_expert_ids(10, ep_size=3, ep_rank=1)
+ ids_2 = compute_local_expert_ids(10, ep_size=3, ep_rank=2)
+
+ assert len(ids_0) == 4
+ assert len(ids_1) == 3
+ assert len(ids_2) == 3
+ # All experts covered, no overlap
+ assert ids_0 | ids_1 | ids_2 == set(range(10))
+ assert ids_0.isdisjoint(ids_1)
+ assert ids_1.isdisjoint(ids_2)
+
+ def test_384_experts_ep8(self):
+ # Kimi-K2.5 config: 384 experts, EP=8
+ for rank in range(8):
+ ids = compute_local_expert_ids(384, ep_size=8, ep_rank=rank)
+ assert len(ids) == 48
+
+ # All experts covered
+ all_ids = set()
+ for rank in range(8):
+ ids = compute_local_expert_ids(384, ep_size=8, ep_rank=rank)
+ all_ids |= ids
+ assert all_ids == set(range(384))
+
+ def test_384_experts_ep16(self):
+ for rank in range(16):
+ ids = compute_local_expert_ids(384, ep_size=16, ep_rank=rank)
+ assert len(ids) == 24
+
+ def test_384_experts_ep24(self):
+ # 384 / 24 = 16 exactly
+ for rank in range(24):
+ ids = compute_local_expert_ids(384, ep_size=24, ep_rank=rank)
+ assert len(ids) == 16
+
+ # round_robin placement tests
+
+ def test_round_robin_basic(self):
+ # 8 experts, EP=2: rank 0 → {0,2,4,6}, rank 1 → {1,3,5,7}
+ rr = "round_robin"
+ ids_0 = compute_local_expert_ids(8, 2, 0, placement=rr)
+ ids_1 = compute_local_expert_ids(8, 2, 1, placement=rr)
+ assert ids_0 == {0, 2, 4, 6}
+ assert ids_1 == {1, 3, 5, 7}
+
+ def test_round_robin_full_coverage(self):
+ # 384 experts, EP=8: all experts covered, no overlap
+ rr = "round_robin"
+ all_ids: set[int] = set()
+ for rank in range(8):
+ ids = compute_local_expert_ids(384, 8, rank, placement=rr)
+ assert ids is not None and len(ids) == 48
+ assert all_ids.isdisjoint(ids)
+ all_ids |= ids
+ assert all_ids == set(range(384))
+
+ def test_round_robin_uneven(self):
+ # 10 experts, EP=3: rank 0→{0,3,6,9}, rank 1→{1,4,7}, rank 2→{2,5,8}
+ rr = "round_robin"
+ ids_0 = compute_local_expert_ids(10, 3, 0, placement=rr)
+ ids_1 = compute_local_expert_ids(10, 3, 1, placement=rr)
+ ids_2 = compute_local_expert_ids(10, 3, 2, placement=rr)
+ assert ids_0 == {0, 3, 6, 9}
+ assert ids_1 == {1, 4, 7}
+ assert ids_2 == {2, 5, 8}
+ assert ids_0 | ids_1 | ids_2 == set(range(10))
+
+
+# ---------------------------------------------------------------------------
+# Unit tests for should_skip_weight
+# ---------------------------------------------------------------------------
+
+
+class TestShouldSkipWeight:
+ def setup_method(self):
+ # Simulate EP=8, rank=0 → experts 0-47
+ self.local_ids = compute_local_expert_ids(384, ep_size=8, ep_rank=0)
+
+ def test_no_filter(self):
+ assert not should_skip_weight("anything", None)
+
+ def test_dense_not_skipped(self):
+ assert not should_skip_weight(
+ "model.layers.0.self_attn.q_proj.weight", self.local_ids
+ )
+
+ def test_local_expert_not_skipped(self):
+ assert not should_skip_weight(
+ "model.layers.0.mlp.experts.10.gate_proj.weight", self.local_ids
+ )
+
+ def test_remote_expert_skipped(self):
+ assert should_skip_weight(
+ "model.layers.0.mlp.experts.200.gate_proj.weight", self.local_ids
+ )
+
+ def test_boundary_expert(self):
+ # Expert 47 is local (last one), 48 is not
+ assert not should_skip_weight(
+ "model.layers.0.mlp.experts.47.gate_proj.weight", self.local_ids
+ )
+ assert should_skip_weight(
+ "model.layers.0.mlp.experts.48.gate_proj.weight", self.local_ids
+ )
+
+ def test_shared_expert_not_skipped(self):
+ assert not should_skip_weight(
+ "model.layers.0.mlp.shared_experts.gate_proj.weight", self.local_ids
+ )
+
+ def test_embedding_not_skipped(self):
+ assert not should_skip_weight("model.embed_tokens.weight", self.local_ids)
+
+ def test_fused_3d_expert_not_skipped(self):
+ # 3D fused-expert tensors (gpt-oss style) have no numeric id.
+ # Must not be skipped — weight_loader handles slicing later.
+ assert not should_skip_weight(
+ "model.layers.0.mlp.experts.gate_proj.weight", self.local_ids
+ )
+
+
+# ---------------------------------------------------------------------------
+# Integration test: safetensors_weights_iterator with EP filtering
+# ---------------------------------------------------------------------------
+
+
+class TestSafetensorsWeightsIteratorWithEpFilter:
+ """Verify that EP filtering produces a strict subset of unfiltered loading
+ and that all expected dense + local expert weights are present."""
+
+ @pytest.fixture(scope="class")
+ def gpt2_files(self):
+ """Download GPT-2 safetensors to a temp dir (shared across class)."""
+ with tempfile.TemporaryDirectory() as tmpdir:
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
+ from vllm.model_executor.model_loader.weight_utils import (
+ download_weights_from_hf,
+ )
+
+ download_weights_from_hf(
+ "openai-community/gpt2",
+ allow_patterns=["*.safetensors"],
+ cache_dir=tmpdir,
+ )
+ files = glob.glob(f"{tmpdir}/**/*.safetensors", recursive=True)
+ assert len(files) > 0
+ yield files
+
+ def test_no_filter_returns_all(self, gpt2_files):
+ """With local_expert_ids=None, all weights are returned (no MoE)."""
+ all_weights = dict(safetensors_weights_iterator(gpt2_files, False))
+ filtered_weights = dict(
+ safetensors_weights_iterator(gpt2_files, False, local_expert_ids=None)
+ )
+ assert set(all_weights.keys()) == set(filtered_weights.keys())
+
+ def test_empty_filter_skips_experts_only(self, gpt2_files):
+ """GPT-2 has no expert weights, so even an empty local_expert_ids
+ set should return all weights (all are dense)."""
+ all_weights = dict(safetensors_weights_iterator(gpt2_files, False))
+ filtered_weights = dict(
+ safetensors_weights_iterator(gpt2_files, False, local_expert_ids=set())
+ )
+ # GPT-2 has no experts, so nothing should be filtered
+ assert set(all_weights.keys()) == set(filtered_weights.keys())
+
+
+class TestEpFilterOnSyntheticMoeWeights:
+ """Create synthetic safetensors files with expert-like naming and verify
+ that the filter correctly skips non-local experts."""
+
+ @pytest.fixture
+ def synthetic_moe_files(self, tmp_path):
+ """Create synthetic safetensors with expert-patterned tensor names."""
+ from safetensors.torch import save_file
+
+ tensors = {}
+ # Dense weights
+ tensors["model.embed_tokens.weight"] = torch.randn(100, 64)
+ tensors["model.layers.0.self_attn.q_proj.weight"] = torch.randn(64, 64)
+ tensors["model.layers.0.input_layernorm.weight"] = torch.randn(64)
+ # Expert weights: 8 experts
+ for expert_id in range(8):
+ tensors[f"model.layers.0.mlp.experts.{expert_id}.gate_proj.weight"] = (
+ torch.randn(128, 64)
+ )
+ tensors[f"model.layers.0.mlp.experts.{expert_id}.up_proj.weight"] = (
+ torch.randn(128, 64)
+ )
+ tensors[f"model.layers.0.mlp.experts.{expert_id}.down_proj.weight"] = (
+ torch.randn(64, 128)
+ )
+ # Shared expert (should never be filtered)
+ tensors["model.layers.0.mlp.shared_experts.gate_proj.weight"] = torch.randn(
+ 128, 64
+ )
+
+ filepath = str(tmp_path / "model-00001-of-00001.safetensors")
+ save_file(tensors, filepath)
+ return [filepath], tensors
+
+ def test_no_filter_returns_all(self, synthetic_moe_files):
+ files, expected = synthetic_moe_files
+ loaded = dict(safetensors_weights_iterator(files, False))
+ assert set(loaded.keys()) == set(expected.keys())
+
+ def test_ep2_rank0_gets_half_experts(self, synthetic_moe_files):
+ files, expected = synthetic_moe_files
+ # EP=2, rank=0 → experts 0-3
+ local_ids = compute_local_expert_ids(8, ep_size=2, ep_rank=0)
+ loaded = dict(
+ safetensors_weights_iterator(files, False, local_expert_ids=local_ids)
+ )
+
+ # Should have all dense + shared + experts 0-3 only
+ for name in loaded:
+ eid = parse_expert_id(name)
+ if eid is not None:
+ assert eid in local_ids, f"Non-local expert {eid} was loaded"
+
+ # Check expert count: 4 experts × 3 weights = 12
+ expert_names = [n for n in loaded if parse_expert_id(n) is not None]
+ assert len(expert_names) == 4 * 3
+
+ # Check all dense weights present
+ assert "model.embed_tokens.weight" in loaded
+ assert "model.layers.0.self_attn.q_proj.weight" in loaded
+ assert "model.layers.0.input_layernorm.weight" in loaded
+ assert "model.layers.0.mlp.shared_experts.gate_proj.weight" in loaded
+
+ def test_ep2_rank1_gets_other_half(self, synthetic_moe_files):
+ files, expected = synthetic_moe_files
+ local_ids = compute_local_expert_ids(8, ep_size=2, ep_rank=1)
+ loaded = dict(
+ safetensors_weights_iterator(files, False, local_expert_ids=local_ids)
+ )
+
+ expert_names = [n for n in loaded if parse_expert_id(n) is not None]
+ assert len(expert_names) == 4 * 3
+ for name in expert_names:
+ assert parse_expert_id(name) in local_ids
+
+ def test_ep8_each_rank_gets_one_expert(self, synthetic_moe_files):
+ files, _ = synthetic_moe_files
+ all_expert_names = set()
+ for rank in range(8):
+ local_ids = compute_local_expert_ids(8, ep_size=8, ep_rank=rank)
+ loaded = dict(
+ safetensors_weights_iterator(files, False, local_expert_ids=local_ids)
+ )
+ expert_names = {n for n in loaded if parse_expert_id(n) is not None}
+ # 1 expert × 3 weights
+ assert len(expert_names) == 3
+ all_expert_names |= expert_names
+
+ # All 8 experts × 3 weights covered across ranks
+ assert len(all_expert_names) == 24
+
+ def test_tensor_values_match(self, synthetic_moe_files):
+ """Filtered tensors have identical values to unfiltered ones."""
+ files, _ = synthetic_moe_files
+ all_weights = dict(safetensors_weights_iterator(files, False))
+
+ local_ids = compute_local_expert_ids(8, ep_size=2, ep_rank=0)
+ filtered = dict(
+ safetensors_weights_iterator(files, False, local_expert_ids=local_ids)
+ )
+
+ for name, tensor in filtered.items():
+ assert torch.equal(tensor, all_weights[name]), f"Tensor mismatch for {name}"
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_modelexpress_loader.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_modelexpress_loader.py
new file mode 100644
index 0000000000000000000000000000000000000000..48eecff1c75d39ec8d0f710c03f2479e5eeac186
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_modelexpress_loader.py
@@ -0,0 +1,133 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import sys
+from types import ModuleType, SimpleNamespace
+
+import pytest
+from torch import nn
+
+from vllm.config import VllmConfig
+from vllm.config.load import LoadConfig
+from vllm.model_executor.model_loader import get_model_loader
+from vllm.model_executor.model_loader.modelexpress_loader import (
+ ModelExpressModelLoader,
+)
+
+
+class FakeModelexpressLoader:
+ calls: list[tuple[str, tuple, dict]] = []
+ loaded_model: nn.Module
+
+ def __init__(self, load_config: LoadConfig):
+ self.load_config = load_config
+
+ def download_model(self, *args, **kwargs):
+ self.calls.append(("download_model", args, kwargs))
+
+ def load_weights(self, *args, **kwargs):
+ self.calls.append(("load_weights", args, kwargs))
+
+ def load_model(self, *args, **kwargs):
+ self.calls.append(("load_model", args, kwargs))
+ return self.loaded_model
+
+
+def _install_fake_modelexpress(monkeypatch):
+ FakeModelexpressLoader.calls = []
+ FakeModelexpressLoader.loaded_model = nn.Module()
+
+ for name in [
+ "modelexpress",
+ "modelexpress.engines",
+ "modelexpress.engines.vllm",
+ ]:
+ monkeypatch.setitem(sys.modules, name, ModuleType(name))
+
+ module = ModuleType("modelexpress.engines.vllm.loader")
+ module.__dict__["MxModelLoader"] = FakeModelexpressLoader
+ monkeypatch.setitem(sys.modules, module.__name__, module)
+
+
+def test_modelexpress_load_format_resolves_to_modelexpress_loader(monkeypatch):
+ _install_fake_modelexpress(monkeypatch)
+
+ loader = get_model_loader(LoadConfig(load_format="modelexpress"))
+
+ assert isinstance(loader, ModelExpressModelLoader)
+
+
+def test_modelexpress_loader_delegates_to_modelexpress(monkeypatch):
+ _install_fake_modelexpress(monkeypatch)
+ loader = ModelExpressModelLoader(LoadConfig(load_format="modelexpress"))
+ model = nn.Module()
+ model_config = SimpleNamespace()
+ vllm_config = SimpleNamespace()
+
+ loader.download_model(model_config)
+ loader.load_weights(model, model_config)
+ FakeModelexpressLoader.loaded_model.train()
+ result = loader.load_model(
+ vllm_config=vllm_config,
+ model_config=model_config,
+ prefix="model",
+ )
+
+ assert result is FakeModelexpressLoader.loaded_model
+ assert not result.training
+ assert FakeModelexpressLoader.calls == [
+ ("download_model", (model_config,), {}),
+ ("load_weights", (model, model_config), {}),
+ (
+ "load_model",
+ (),
+ {
+ "vllm_config": vllm_config,
+ "model_config": model_config,
+ "prefix": "model",
+ },
+ ),
+ ]
+
+
+def test_modelexpress_loader_missing_modelexpress_error(monkeypatch):
+ import importlib
+
+ def missing_modelexpress(name):
+ raise ModuleNotFoundError(name=name)
+
+ monkeypatch.setattr(importlib, "import_module", missing_modelexpress)
+
+ with pytest.raises(ImportError, match="requires the ModelExpress Python package"):
+ ModelExpressModelLoader(LoadConfig(load_format="modelexpress"))
+
+
+def test_modelexpress_loader_preserves_internal_import_errors(monkeypatch):
+ import importlib
+
+ def missing_dependency(name):
+ raise ModuleNotFoundError(name="not_modelexpress_dependency")
+
+ monkeypatch.setattr(importlib, "import_module", missing_dependency)
+
+ with pytest.raises(ModuleNotFoundError) as exc_info:
+ ModelExpressModelLoader(LoadConfig(load_format="modelexpress"))
+ assert exc_info.value.name == "not_modelexpress_dependency"
+
+
+def test_modelexpress_load_format_allows_object_storage_model_weights():
+ model_config = SimpleNamespace(
+ architecture="UnknownForTest",
+ config_updated=False,
+ convert_type=None,
+ is_hybrid=False,
+ model="test-model",
+ model_weights="s3://bucket/model",
+ )
+ vllm_config = object.__new__(VllmConfig)
+ vllm_config.model_config = model_config
+ vllm_config.load_config = LoadConfig(load_format="modelexpress")
+
+ vllm_config.try_verify_and_update_config()
+
+ assert vllm_config.load_config.load_format == "modelexpress"
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_registry.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_registry.py
new file mode 100644
index 0000000000000000000000000000000000000000..9569c0663651bbccc646833a3d73eacf95bc1212
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_registry.py
@@ -0,0 +1,90 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+from torch import nn
+
+from vllm.config import ModelConfig
+from vllm.config.load import LoadConfig
+from vllm.model_executor.model_loader import get_model_loader, register_model_loader
+from vllm.model_executor.model_loader.base_loader import BaseModelLoader
+from vllm.model_executor.model_loader.default_loader import DefaultModelLoader
+
+
+@register_model_loader("custom_load_format")
+class CustomModelLoader(BaseModelLoader):
+ def __init__(self, load_config: LoadConfig) -> None:
+ super().__init__(load_config)
+
+ def download_model(self, model_config: ModelConfig) -> None:
+ pass
+
+ def load_weights(self, model: nn.Module, model_config: ModelConfig) -> None:
+ pass
+
+
+def test_register_model_loader():
+ load_config = LoadConfig(load_format="custom_load_format")
+ assert isinstance(get_model_loader(load_config), CustomModelLoader)
+
+
+def test_invalid_model_loader():
+ with pytest.raises(ValueError):
+
+ @register_model_loader("invalid_load_format")
+ class InValidModelLoader:
+ pass
+
+
+def test_default_loader_rejects_zero_num_threads():
+ # num_threads=0 used to fail late in ThreadPoolExecutor ("max_workers must be > 0").
+ with pytest.raises(ValueError, match="num_threads"):
+ DefaultModelLoader(
+ LoadConfig(
+ model_loader_extra_config={
+ "enable_multithread_load": True,
+ "num_threads": 0,
+ }
+ )
+ )
+
+
+def test_default_loader_rejects_multithread_with_non_lazy_strategy():
+ # The multi-thread loader ignores safetensors_load_strategy; reject the
+ # combination instead of silently dropping the requested strategy.
+ with pytest.raises(ValueError, match="does not support"):
+ DefaultModelLoader(
+ LoadConfig(
+ safetensors_load_strategy="torchao",
+ model_loader_extra_config={"enable_multithread_load": True},
+ )
+ )
+
+
+def test_default_loader_explicit_safetensors_does_not_misread_pt(tmp_path):
+ # Explicit safetensors must not fall back to a .pt and open it as safetensors.
+ (tmp_path / "model.pt").write_bytes(b"\x00\x00\x00\x00")
+ loader = DefaultModelLoader(LoadConfig(load_format="safetensors"))
+ with pytest.raises(RuntimeError, match="Cannot find any model weights"):
+ loader._prepare_weights(
+ str(tmp_path),
+ None,
+ None,
+ fall_back_to_pt=True,
+ allow_patterns_overrides=None,
+ )
+
+
+def test_default_loader_hf_still_falls_back_to_pt(tmp_path):
+ # Control: load_format="hf" still picks up .pt weights via fallback.
+ (tmp_path / "model.pt").write_bytes(b"\x00\x00\x00\x00")
+ loader = DefaultModelLoader(LoadConfig(load_format="hf"))
+ _, files, use_safetensors = loader._prepare_weights(
+ str(tmp_path),
+ None,
+ None,
+ fall_back_to_pt=True,
+ allow_patterns_overrides=None,
+ )
+ assert use_safetensors is False
+ assert any(f.endswith("model.pt") for f in files)
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_reload.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_reload.py
new file mode 100644
index 0000000000000000000000000000000000000000..b564f495c16382b30176c8b7fdd61d2398dd6924
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_reload.py
@@ -0,0 +1,498 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import gc
+import inspect
+from weakref import WeakKeyDictionary, ref
+
+import pytest
+import torch
+from torch.nn.parameter import UninitializedParameter
+
+import vllm.model_executor.model_loader.reload.meta as reload_meta
+from vllm.model_executor.layers.linear import QKVParallelLinear
+from vllm.model_executor.model_loader.reload.layerwise import (
+ finalize_layerwise_reload,
+ initialize_layerwise_reload,
+ record_metadata_for_reloading,
+)
+from vllm.model_executor.model_loader.reload.meta import (
+ capture_layer_to_meta,
+ get_numel_loaded,
+ materialize_layer,
+ materialize_meta_tensor,
+ restore_layer_on_meta,
+ to_meta_tensor,
+)
+from vllm.model_executor.model_loader.reload.types import LayerReloadingInfo
+from vllm.model_executor.model_loader.reload.utils import get_layer_tensors
+from vllm.model_executor.model_loader.weight_utils import (
+ composed_weight_loader,
+ default_weight_loader,
+)
+from vllm.platforms import current_platform
+
+
+def _fp8_reload_unsupported() -> bool:
+ """Whether the FP8 reload/online-quantize tests should be skipped.
+
+ ``supports_fp8()`` returns True on MI250 (gfx90a) because the general
+ quantization paths upcast FP8 weights, but gfx90a has no native FP8 and
+ cannot run these reload models, so treat it as unsupported here.
+ """
+ if not current_platform.supports_fp8():
+ return True
+ if current_platform.is_rocm():
+ from vllm.platforms.rocm import on_gfx90a
+
+ return on_gfx90a()
+ return False
+
+
+class _AliasedBufferLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ weight = torch.arange(6, dtype=torch.float32).reshape(2, 3)
+ self.weight = torch.nn.Parameter(weight)
+ self.register_buffer(
+ "weight_view", self.weight.detach().view(-1), persistent=False
+ )
+
+
+class _ParentAliasedChildBufferLayer(torch.nn.Module):
+ def __init__(self):
+ super().__init__()
+ self.scale = torch.nn.Parameter(torch.ones(1))
+ self.conv1d = torch.nn.Linear(3, 2, bias=False)
+ self.conv1d.weight.data.copy_(
+ torch.arange(6, dtype=torch.float32).reshape(2, 3)
+ )
+ self.register_buffer(
+ "conv_weights", self.conv1d.weight.detach().view(-1), persistent=False
+ )
+
+
+class _AliasedBufferWithUninitializedChildLayer(_AliasedBufferLayer):
+ def __init__(self):
+ super().__init__()
+ self.child = torch.nn.Module()
+ self.child.register_parameter(
+ "lazy_weight", UninitializedParameter(requires_grad=False)
+ )
+
+
+def test_move_metatensors():
+ tensor = torch.empty((1, 2, 3))
+ meta_tensor = to_meta_tensor(tensor)
+ materialized_tensor = materialize_meta_tensor(meta_tensor)
+
+ assert meta_tensor.device.type == "meta"
+ assert tensor.device == materialized_tensor.device
+
+ assert tensor.dtype == meta_tensor.dtype == materialized_tensor.dtype
+ assert tensor.shape == meta_tensor.shape == materialized_tensor.shape
+ assert tensor.__class__ == meta_tensor.__class__ == materialized_tensor.__class__
+ assert tensor.__dict__ == meta_tensor.__dict__ == materialized_tensor.__dict__
+
+
+def test_reload_lifecycle():
+ layer = torch.nn.Linear(2, 3)
+ info = LayerReloadingInfo(
+ restore_metadata=capture_layer_to_meta(layer),
+ restore_device=torch.device("cpu"),
+ )
+
+ restore_layer_on_meta(layer, info)
+ for name, tensor in get_layer_tensors(layer).items():
+ meta_tensor = getattr(layer, name)
+ assert tensor.dtype == meta_tensor.dtype
+ assert tensor.shape == meta_tensor.shape
+ assert tensor.__class__ == meta_tensor.__class__
+ assert tensor.__dict__ == meta_tensor.__dict__
+
+ materialize_layer(layer, info)
+ for name, tensor in get_layer_tensors(layer).items():
+ materialized_tensor = getattr(layer, name)
+ assert tensor.dtype == materialized_tensor.dtype
+ assert tensor.shape == materialized_tensor.shape
+ assert tensor.__class__ == materialized_tensor.__class__
+ assert tensor.__dict__ == materialized_tensor.__dict__
+
+
+def test_materialize_layer_preserves_non_meta_tensors():
+ """Ensure that materialize_layer does not overwrite non meta tensors."""
+ layer = torch.nn.Linear(2, 3, bias=True)
+
+ # Create a non meta bias tensor and meta weight, which can happen with FP8
+ bias_values = torch.ones(3)
+ layer.bias.data.copy_(bias_values)
+ layer.weight = torch.nn.Parameter(layer.weight.data.to("meta"))
+
+ assert layer.weight.is_meta
+ assert not layer.bias.is_meta
+
+ # materialize the layer weights after the bias is initialized
+ info = LayerReloadingInfo(
+ restore_metadata=({}, {}),
+ restore_device=torch.device("cpu"),
+ )
+ materialize_layer(layer, info)
+
+ # Ensure the weight materialized off meta
+ assert not layer.weight.is_meta
+ assert layer.weight.device.type == "cpu"
+
+ # Ensure that the bias is (still) not meta and values are unchanged
+ assert not layer.bias.is_meta
+ assert torch.equal(layer.bias.data, bias_values)
+
+
+def test_model_cleanup(dist_init, default_vllm_config):
+ layer = QKVParallelLinear(2, 3, 4)
+ assert layer.weight.weight_loader.__self__ is layer
+ info = LayerReloadingInfo(
+ restore_metadata=capture_layer_to_meta(layer),
+ restore_device=torch.device("cpu"),
+ )
+
+ mock_info_dict: WeakKeyDictionary[torch.nn.Module, LayerReloadingInfo] = (
+ WeakKeyDictionary()
+ )
+ mock_info_dict[layer] = info
+ layer_ref = ref(layer)
+
+ del layer
+ gc.collect()
+
+ assert layer_ref() is None
+ assert len(mock_info_dict) == 0
+
+
+def test_get_numel_loaded():
+ param = torch.empty(10, device="meta")
+ loaded_weight = torch.empty(10)
+
+ def complex_weight_loader(param, loaded_weight):
+ param[:3] = loaded_weight[:3]
+ param[5:8] = loaded_weight[5:8]
+ return "value"
+
+ args = inspect.signature(complex_weight_loader).bind(param, loaded_weight)
+ num_loaded, ret = get_numel_loaded(complex_weight_loader, args)
+ assert num_loaded == 6
+ assert ret == "value"
+
+
+def test_get_numel_loaded_caps_at_param_size():
+ # composed_weight_loader copies into the param twice (the load and the
+ # in-place post-load transform), but only param.numel() distinct elements
+ # are loaded. get_numel_loaded must not double-count, otherwise a layer's
+ # loaded-element total can be reached early and trailing params get dropped.
+ param = torch.empty(10)
+ loaded_weight = torch.ones(10)
+ loader = composed_weight_loader(default_weight_loader, lambda x: x + 1)
+
+ args = inspect.signature(loader).bind(param, loaded_weight)
+ num_loaded, _ = get_numel_loaded(loader, args)
+ assert num_loaded == 10
+
+
+class _ComposedLoaderLayer(torch.nn.Module):
+ """Mimics a Mamba2 mixer's equal-numel direct params (A, D, dt_bias).
+
+ ``A`` uses ``composed_weight_loader`` (an extra in-place transform copy),
+ matching ``MambaMixer2`` where ``A`` is loaded as ``-exp(A_log)``.
+ """
+
+ def __init__(self):
+ super().__init__()
+ self.A = torch.nn.Parameter(torch.empty(4, dtype=torch.float32))
+ self.D = torch.nn.Parameter(torch.ones(4))
+ self.dt_bias = torch.nn.Parameter(torch.ones(4))
+ self.A.weight_loader = composed_weight_loader(
+ default_weight_loader, lambda x: -torch.exp(x.float())
+ )
+ self.D.weight_loader = default_weight_loader
+ self.dt_bias.weight_loader = default_weight_loader
+
+
+def test_layerwise_reload_composed_loader_does_not_drop_params(monkeypatch):
+ # Regression test: a composed_weight_loader param (A) used to double-count
+ # its elements, finalizing the layer before the trailing param (D) was
+ # loaded and leaving it as uninitialized materialized memory.
+ layer = _ComposedLoaderLayer()
+ model = torch.nn.Sequential(layer)
+
+ def materialize_with_sentinel(meta_tensor):
+ tensor = torch.empty_strided(
+ size=tuple(meta_tensor.size()),
+ stride=tuple(meta_tensor.stride()),
+ dtype=meta_tensor.dtype,
+ requires_grad=False,
+ )
+ tensor.fill_(float("nan"))
+ tensor.__class__ = meta_tensor.__class__
+ tensor.__dict__ = meta_tensor.__dict__.copy()
+ return tensor
+
+ monkeypatch.setattr(
+ reload_meta, "materialize_meta_tensor", materialize_with_sentinel
+ )
+
+ loaded = {
+ "A": torch.full((4,), 0.5),
+ "dt_bias": torch.full((4,), 3.0),
+ "D": torch.full((4,), 7.0),
+ }
+
+ record_metadata_for_reloading(model)
+ initialize_layerwise_reload(model)
+ # Mimic real load_weights: resolve params once, then load in checkpoint
+ # order with D last (the param that was dropped).
+ params = dict(layer.named_parameters())
+ for name in ("A", "dt_bias", "D"):
+ param = params[name]
+ param.weight_loader(param, loaded[name])
+ finalize_layerwise_reload(model, model_config=None)
+
+ assert torch.equal(layer.A, -torch.exp(loaded["A"]))
+ assert torch.equal(layer.dt_bias, loaded["dt_bias"])
+ assert torch.equal(layer.D, loaded["D"])
+
+
+def test_layerwise_reload_skips_non_persistent_parameter_alias_buffers(monkeypatch):
+ layer = _AliasedBufferLayer()
+ model = torch.nn.Sequential(layer)
+ loaded_weight = torch.full_like(layer.weight, 7.0)
+
+ def materialize_with_sentinel(meta_tensor):
+ tensor = torch.empty_strided(
+ size=tuple(meta_tensor.size()),
+ stride=tuple(meta_tensor.stride()),
+ dtype=meta_tensor.dtype,
+ requires_grad=False,
+ )
+ tensor.fill_(-123.0)
+ tensor.__class__ = meta_tensor.__class__
+ tensor.__dict__ = meta_tensor.__dict__.copy()
+ return tensor
+
+ monkeypatch.setattr(
+ reload_meta, "materialize_meta_tensor", materialize_with_sentinel
+ )
+
+ record_metadata_for_reloading(model)
+ initialize_layerwise_reload(model)
+ layer.weight.weight_loader(layer.weight, loaded_weight)
+ finalize_layerwise_reload(model, model_config=None)
+
+ assert torch.equal(layer.weight, loaded_weight)
+ assert layer.weight_view.untyped_storage().data_ptr() == (
+ layer.weight.untyped_storage().data_ptr()
+ )
+
+
+def test_capture_layer_to_meta_skips_uninitialized_parameter_storage_ptrs():
+ layer = _AliasedBufferWithUninitializedChildLayer()
+
+ _, buffers = capture_layer_to_meta(layer)
+
+ assert "weight_view" not in buffers
+
+
+def test_layerwise_reload_skips_child_parameter_alias_buffers(monkeypatch):
+ layer = _ParentAliasedChildBufferLayer()
+ model = torch.nn.Sequential(layer)
+ loaded_conv = torch.full_like(layer.conv1d.weight, 7.0)
+ loaded_scale = torch.full_like(layer.scale, 3.0)
+
+ def materialize_with_sentinel(meta_tensor):
+ tensor = torch.empty_strided(
+ size=tuple(meta_tensor.size()),
+ stride=tuple(meta_tensor.stride()),
+ dtype=meta_tensor.dtype,
+ requires_grad=False,
+ )
+ tensor.fill_(-123.0)
+ tensor.__class__ = meta_tensor.__class__
+ tensor.__dict__ = meta_tensor.__dict__.copy()
+ return tensor
+
+ monkeypatch.setattr(
+ reload_meta, "materialize_meta_tensor", materialize_with_sentinel
+ )
+
+ record_metadata_for_reloading(model)
+ initialize_layerwise_reload(model)
+ layer.conv1d.weight.weight_loader(layer.conv1d.weight, loaded_conv)
+ layer.scale.weight_loader(layer.scale, loaded_scale)
+ finalize_layerwise_reload(model, model_config=None)
+
+ assert torch.equal(layer.conv1d.weight, loaded_conv)
+ assert torch.equal(layer.conv_weights, loaded_conv.view(-1))
+ assert layer.conv_weights.untyped_storage().data_ptr() == (
+ layer.conv1d.weight.untyped_storage().data_ptr()
+ )
+
+
+@pytest.mark.parametrize(
+ "tp_size", [pytest.param(1), pytest.param(2, marks=[pytest.mark.slow_test])]
+)
+@pytest.mark.parametrize(
+ "base_model,mul_model,add_model",
+ [
+ pytest.param(
+ "Qwen/Qwen3-0.6B",
+ "inference-optimization/Qwen3-0.6B-debug-multiply",
+ "inference-optimization/Qwen3-0.6B-debug-add",
+ marks=[pytest.mark.slow_test],
+ ),
+ pytest.param(
+ "inference-optimization/Qwen3-0.6B-FP8_BLOCK",
+ "inference-optimization/Qwen3-0.6B-debug-multiply-FP8_BLOCK",
+ "inference-optimization/Qwen3-0.6B-debug-add-FP8_BLOCK",
+ marks=[pytest.mark.slow_test],
+ ),
+ pytest.param(
+ "inference-optimization/Qwen3-0.6B-W4A16-G128",
+ "inference-optimization/Qwen3-0.6B-debug-multiply-W4A16-G128",
+ "inference-optimization/Qwen3-0.6B-debug-add-W4A16-G128",
+ marks=[pytest.mark.slow_test],
+ ),
+ pytest.param(
+ "inference-optimization/DeepSeek-V3-debug-empty",
+ "inference-optimization/DeepSeek-V3-debug-multiply",
+ "inference-optimization/DeepSeek-V3-debug-add",
+ marks=[pytest.mark.slow_test],
+ ),
+ pytest.param(
+ "inference-optimization/DeepSeek-V3-debug-empty-FP8_DYNAMIC",
+ "inference-optimization/DeepSeek-V3-debug-multiply-FP8_DYNAMIC",
+ "inference-optimization/DeepSeek-V3-debug-add-FP8_DYNAMIC",
+ ),
+ pytest.param(
+ "inference-optimization/DeepSeek-V3-debug-empty-NVFP4A16",
+ "inference-optimization/DeepSeek-V3-debug-multiply-NVFP4A16",
+ "inference-optimization/DeepSeek-V3-debug-add-NVFP4A16",
+ marks=[pytest.mark.slow_test],
+ ),
+ ],
+)
+def test_reload_weights(base_model, mul_model, add_model, tp_size, vllm_runner):
+ if current_platform.device_count() < tp_size:
+ pytest.skip(reason="Not enough CUDA devices")
+
+ if "FP8" in base_model and _fp8_reload_unsupported():
+ pytest.skip(reason="Requires FP8 support")
+
+ with vllm_runner(
+ model_name=base_model,
+ tensor_parallel_size=tp_size,
+ enable_expert_parallel=(tp_size > 1 and "DeepSeek" in base_model),
+ enable_prefix_caching=False,
+ max_model_len=16,
+ max_num_seqs=1,
+ ) as llm:
+ llm.collective_rpc("reload_weights", kwargs={"weights_path": mul_model})
+ mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
+ add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
+ assert mul_perp < add_perp
+
+ llm.collective_rpc("reload_weights", kwargs={"weights_path": add_model})
+ mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
+ add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
+ assert add_perp < mul_perp
+
+
+def test_kv_scale_reload(vllm_runner):
+ """Test reloading a checkpoint that contains k_scale/v_scale weights."""
+ if _fp8_reload_unsupported():
+ pytest.skip(reason="Requires FP8 support")
+
+ model = "nm-testing/Llama-3.2-1B-Instruct-FP8-KV"
+
+ # Load dummy weights, then reload real checkpoint
+ with vllm_runner(
+ model_name=model,
+ load_format="dummy",
+ enable_prefix_caching=False,
+ max_model_len=16,
+ max_num_seqs=1,
+ ) as llm:
+ llm.collective_rpc(
+ "update_config",
+ kwargs={"overrides": {"load_config": {"load_format": "auto"}}},
+ )
+ llm.collective_rpc("reload_weights", kwargs={"weights_path": model})
+ reloaded_perp = llm.generate_prompt_perplexity(
+ ["The capital of France is the city of Paris"],
+ mask=["The capital of France is"],
+ )[0]
+
+ assert reloaded_perp < 10
+
+
+@pytest.mark.parametrize(
+ "tp_size", [pytest.param(1), pytest.param(2, marks=[pytest.mark.slow_test])]
+)
+@pytest.mark.parametrize(
+ "base_model,mul_model,add_model,quantization",
+ [
+ pytest.param(
+ "Qwen/Qwen3-0.6B",
+ "inference-optimization/Qwen3-0.6B-debug-multiply",
+ "inference-optimization/Qwen3-0.6B-debug-add",
+ "fp8",
+ ),
+ pytest.param(
+ "inference-optimization/DeepSeek-V3-debug-empty",
+ "inference-optimization/DeepSeek-V3-debug-multiply",
+ "inference-optimization/DeepSeek-V3-debug-add",
+ "fp8",
+ marks=[pytest.mark.slow_test],
+ ),
+ pytest.param(
+ "Qwen/Qwen3-0.6B",
+ "inference-optimization/Qwen3-0.6B-debug-multiply",
+ "inference-optimization/Qwen3-0.6B-debug-add",
+ "mxfp8",
+ marks=[pytest.mark.slow_test],
+ ),
+ pytest.param(
+ "inference-optimization/DeepSeek-V3-debug-empty",
+ "inference-optimization/DeepSeek-V3-debug-multiply",
+ "inference-optimization/DeepSeek-V3-debug-add",
+ "mxfp8",
+ marks=[
+ pytest.mark.slow_test,
+ pytest.mark.xfail(reason="mxfp4 & mla is not supported yet"),
+ ],
+ ),
+ ],
+)
+def test_online_quantize_reload(
+ base_model, mul_model, add_model, quantization, tp_size, vllm_runner
+):
+ if current_platform.device_count() < tp_size:
+ pytest.skip(reason="Not enough GPU devices")
+
+ if quantization == "fp8" and _fp8_reload_unsupported():
+ pytest.skip(reason="Requires FP8 support")
+
+ with vllm_runner(
+ model_name=base_model,
+ quantization=quantization,
+ tensor_parallel_size=tp_size,
+ enable_expert_parallel=(tp_size > 1 and "DeepSeek" in base_model),
+ enable_prefix_caching=False,
+ max_model_len=16,
+ max_num_seqs=1,
+ ) as llm:
+ llm.collective_rpc("reload_weights", kwargs={"weights_path": mul_model})
+ mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
+ add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
+ assert mul_perp < add_perp
+
+ llm.collective_rpc("reload_weights", kwargs={"weights_path": add_model})
+ mul_perp = llm.generate_prompt_perplexity(["3 4 = 12"], mask=["3 4 ="])[0]
+ add_perp = llm.generate_prompt_perplexity(["3 4 = 7"], mask=["3 4 ="])[0]
+ assert add_perp < mul_perp
diff --git a/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_sharded_state_loader.py b/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_sharded_state_loader.py
new file mode 100644
index 0000000000000000000000000000000000000000..a2d7bb4fb59f2a7e23d3696a18dce1554ba47a82
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/model_loader/test_sharded_state_loader.py
@@ -0,0 +1,165 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import fnmatch
+import multiprocessing as mp
+import os
+import shutil
+from tempfile import TemporaryDirectory
+
+import pytest
+import torch
+from huggingface_hub import snapshot_download
+
+from vllm import LLM, SamplingParams
+from vllm.model_executor.model_loader import ShardedStateLoader
+from vllm.platforms import current_platform
+
+prompts = [
+ "Hello, my name is",
+ "The president of the United States is",
+ "The capital of France is",
+ "The future of AI is",
+]
+
+# Create a sampling params object.
+sampling_params = SamplingParams(
+ temperature=0,
+ max_tokens=256,
+ ignore_eos=True,
+)
+
+
+def test_filter_subtensors():
+ state_dict = {
+ "a": torch.empty(2),
+ "b": torch.empty((2, 4)),
+ "c": torch.empty((2, 4, 8)),
+ }
+ state_dict.update(
+ {
+ "x": state_dict["b"],
+ "y": state_dict["c"][1, 2, :],
+ "z": state_dict["c"][1, :, 4],
+ }
+ )
+ filtered_state_dict = ShardedStateLoader._filter_subtensors(state_dict)
+ assert tuple(filtered_state_dict.keys()) == ("a", "b", "c")
+ for key, tensor in filtered_state_dict.items():
+ # NOTE: don't use `equal` here, as the tensor might contain NaNs
+ assert tensor is state_dict[key]
+
+
+@pytest.fixture(scope="module")
+def llama_3p2_1b_files():
+ input_dir = snapshot_download(
+ "meta-llama/Llama-3.2-1B-Instruct", ignore_patterns=["*.bin*", "original/*"]
+ )
+
+ yield input_dir
+
+
+def _run_writer(input_dir, output_dir, weights_patterns, **kwargs):
+ llm_sharded_writer = LLM(model=input_dir, **kwargs)
+
+ # Dump worker states to output directory
+ llm_sharded_writer.llm_engine.engine_core.save_sharded_state(path=output_dir)
+
+ # Copy metadata files to output directory
+ for file in os.listdir(input_dir):
+ if os.path.isdir(os.path.join(input_dir, file)):
+ shutil.copytree(
+ os.path.join(input_dir, file), os.path.join(output_dir, file)
+ )
+ elif not any(fnmatch.fnmatch(file, ext) for ext in weights_patterns):
+ shutil.copy(os.path.join(input_dir, file), output_dir)
+
+
+def _run_generate(input_dir, queue: mp.Queue, **kwargs):
+ llm = LLM(model=input_dir, **kwargs)
+ gen = llm.generate(prompts, sampling_params)
+ queue.put([g.outputs[0].__dict__ for g in gen])
+ queue.close()
+ queue.join_thread()
+
+
+@pytest.mark.parametrize("enable_lora", [False, True])
+@pytest.mark.parametrize("tp_size", [1, 2])
+def test_sharded_state_loader(
+ enable_lora, tp_size, num_gpus_available, llama_3p2_1b_files
+):
+ if num_gpus_available < tp_size:
+ pytest.skip(f"Not enough GPUs for tensor parallelism {tp_size}")
+
+ weights_patterns = ("*.safetensors",)
+ gpu_memory_utilization = 0.8
+ input_dir = llama_3p2_1b_files
+ ctx = mp.get_context("spawn")
+
+ platform_args = {}
+ if current_platform.is_rocm() or current_platform.is_xpu():
+ platform_args["max_num_seqs"] = 1
+
+ # Run in separate processes for memory & CUDA isolation
+ with TemporaryDirectory() as output_dir:
+ p = ctx.Process(
+ target=_run_writer,
+ args=(input_dir, output_dir, weights_patterns),
+ kwargs=dict(
+ tensor_parallel_size=tp_size,
+ gpu_memory_utilization=gpu_memory_utilization,
+ enforce_eager=True,
+ **platform_args,
+ ),
+ )
+ p.start()
+ p.join()
+
+ queue = ctx.Queue()
+
+ p = ctx.Process(
+ target=_run_generate,
+ args=(input_dir, queue),
+ kwargs=dict(
+ enable_lora=enable_lora,
+ gpu_memory_utilization=gpu_memory_utilization,
+ tensor_parallel_size=tp_size,
+ **platform_args,
+ ),
+ )
+ p.start()
+ # Call queue.get() before p.join() to prevent deadlock:
+ # If p.join() is called before queue.get() and the queue is full,
+ # the child process may block while writing to the queue and never
+ # terminate, causing the parent to wait indefinitely on p.join().
+ # See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
+ out_before = queue.get()
+ p.join()
+ queue.close()
+ queue.join_thread()
+
+ queue = ctx.Queue()
+
+ p = ctx.Process(
+ target=_run_generate,
+ args=(output_dir, queue),
+ kwargs=dict(
+ enable_lora=enable_lora,
+ gpu_memory_utilization=gpu_memory_utilization,
+ tensor_parallel_size=tp_size,
+ load_format="sharded_state",
+ **platform_args,
+ ),
+ )
+ p.start()
+ # Call queue.get() before p.join() to prevent deadlock:
+ # If p.join() is called before queue.get() and the queue is full,
+ # the child process may block while writing to the queue and never
+ # terminate, causing the parent to wait indefinitely on p.join().
+ # See: https://github.com/vllm-project/vllm/pull/22371#discussion_r2257773814
+ out_after = queue.get()
+ p.join()
+ queue.close()
+ queue.join_thread()
+
+ assert out_before == out_after
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py b/asian_ai/performance/vLLM/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py
new file mode 100644
index 0000000000000000000000000000000000000000..0c4c52f7ac04a2b72cf4155590a97e2c2983b112
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_cpu_unquantized_gemm_dispatch.py
@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for CPU unquantized GEMM dispatch behavior."""
+
+import pytest
+import torch
+
+from vllm.model_executor.layers import utils
+from vllm.platforms import current_platform
+
+
+@pytest.fixture(scope="module")
+def _mock_zentorch_linear_unary():
+ """Register a mock zentorch_linear_unary op when zentorch is not installed.
+
+ Allows the dispatch tests to run in CI without a real zentorch build.
+ Skips registration when zentorch is already available.
+ """
+ if hasattr(torch.ops.zentorch, "zentorch_linear_unary"):
+ yield
+ return
+
+ lib_def = torch.library.Library("zentorch", "DEF")
+ lib_def.define(
+ "zentorch_linear_unary("
+ "Tensor input, "
+ "Tensor weight, "
+ "Tensor? bias, "
+ "bool is_weight_prepacked=False"
+ ") -> Tensor"
+ )
+
+ lib_impl = torch.library.Library("zentorch", "IMPL", "CPU")
+ lib_impl.impl(
+ "zentorch_linear_unary",
+ lambda input, weight, bias, is_weight_prepacked=False: (
+ torch.nn.functional.linear(input, weight, bias)
+ ),
+ )
+
+ yield
+
+ lib_impl._destroy()
+ lib_def._destroy()
+
+
+@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
+def test_dispatch_cpu_unquantized_gemm_uses_zentorch_on_zen(monkeypatch):
+ monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
+
+ layer = torch.nn.Linear(16, 8, bias=True)
+ x = torch.randn(4, 16)
+ expected = torch.nn.functional.linear(x, layer.weight, layer.bias)
+
+ utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
+ output = layer.cpu_linear(x, layer.weight, layer.bias)
+
+ torch.testing.assert_close(output, expected)
+
+
+@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
+def test_dispatch_cpu_unquantized_gemm_zen_remove_weight(monkeypatch):
+ monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
+
+ layer = torch.nn.Linear(16, 8, bias=True)
+ utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=True)
+
+ assert layer.weight.numel() == 0
+
+
+@pytest.mark.usefixtures("_mock_zentorch_linear_unary")
+def test_dispatch_cpu_unquantized_gemm_logs_zentorch_dispatch(monkeypatch):
+ monkeypatch.setattr(current_platform, "is_zen_cpu", lambda: True)
+ expected_prepacked = bool(utils.envs.VLLM_ZENTORCH_WEIGHT_PREPACK) and hasattr(
+ torch.ops.zentorch, "zentorch_weight_prepack_for_linear"
+ )
+
+ log_calls = []
+ monkeypatch.setattr(
+ utils.logger, "debug_once", lambda *args: log_calls.append(args)
+ )
+
+ layer = torch.nn.Linear(16, 8, bias=True)
+ utils.dispatch_cpu_unquantized_gemm(layer, remove_weight=False)
+
+ assert log_calls == [
+ (
+ "CPU unquantized GEMM dispatch: using zentorch_linear_unary (prepacked=%s)",
+ expected_prepacked,
+ )
+ ]
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_eagle_quantization.py b/asian_ai/performance/vLLM/tests/model_executor/test_eagle_quantization.py
new file mode 100644
index 0000000000000000000000000000000000000000..4c99889d87515b94197f445b5aaf161d33a162dc
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_eagle_quantization.py
@@ -0,0 +1,159 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from unittest.mock import Mock, patch
+
+import pytest
+import torch
+
+from vllm.config import LoadConfig, ModelConfig, SpeculativeConfig, VllmConfig
+from vllm.model_executor.models.utils import get_draft_quant_config
+from vllm.platforms import current_platform
+
+DEVICE_TYPE = current_platform.device_type
+DEVICES = (
+ [f"{DEVICE_TYPE}:{i}" for i in range(min(torch.accelerator.device_count(), 2))]
+ if not current_platform.is_cpu()
+ else ["cpu"]
+)
+
+
+def test_get_draft_quant_config_with_draft_model():
+ mock_draft_model_config = Mock(spec=ModelConfig)
+ mock_load_config = Mock(spec=LoadConfig)
+ mock_speculative_config = Mock(spec=SpeculativeConfig)
+ mock_speculative_config.draft_model_config = mock_draft_model_config
+
+ mock_vllm_config = Mock(spec=VllmConfig)
+ mock_vllm_config.speculative_config = mock_speculative_config
+ mock_vllm_config.load_config = mock_load_config
+
+ mock_quant_config = Mock()
+ with patch.object(
+ VllmConfig, "get_quantization_config", return_value=mock_quant_config
+ ):
+ result = get_draft_quant_config(mock_vllm_config)
+
+ # Verify the function calls get_quantization_config with draft model config
+ VllmConfig.get_quantization_config.assert_called_once_with(
+ mock_draft_model_config, mock_load_config
+ )
+ assert result == mock_quant_config
+
+
+def test_get_draft_quant_config_without_draft_model():
+ mock_speculative_config = Mock(spec=SpeculativeConfig)
+ mock_speculative_config.draft_model_config = None
+
+ mock_vllm_config = Mock(spec=VllmConfig)
+ mock_vllm_config.speculative_config = mock_speculative_config
+ mock_vllm_config.load_config = Mock(spec=LoadConfig)
+
+ result = get_draft_quant_config(mock_vllm_config)
+
+ assert result is None
+
+
+@torch.inference_mode()
+@pytest.mark.parametrize("device", DEVICES)
+def test_fc_layer_quant_config_usage(default_vllm_config, dist_init, device) -> None:
+ import torch
+
+ from vllm.model_executor.layers.linear import ReplicatedLinear
+
+ if current_platform.is_cuda_alike():
+ torch.accelerator.set_device_index(device)
+
+ torch.set_default_device(device)
+
+ input_size = 256
+ output_size = 128
+
+ fc_no_quant = ReplicatedLinear(
+ input_size=input_size,
+ output_size=output_size,
+ bias=False,
+ params_dtype=torch.float16,
+ quant_config=None,
+ prefix="fc",
+ )
+
+ assert fc_no_quant.quant_config is None
+ assert fc_no_quant.input_size == input_size
+ assert fc_no_quant.output_size == output_size
+
+ mock_quant_config = Mock()
+ fc_with_quant = ReplicatedLinear(
+ input_size=input_size,
+ output_size=output_size,
+ bias=False,
+ params_dtype=torch.float16,
+ quant_config=mock_quant_config,
+ prefix="fc",
+ )
+
+ assert fc_with_quant.quant_config == mock_quant_config
+
+ # Check forward pass
+ x = torch.randn(2, input_size, dtype=torch.float16)
+ output, _ = fc_no_quant(x)
+ assert output.shape == (2, output_size)
+
+
+def test_maybe_remap_kv_scale_name():
+ from vllm.model_executor.model_loader.weight_utils import maybe_remap_kv_scale_name
+
+ params_dict = {
+ "layers.0.self_attn.kv_scale": Mock(),
+ "layers.1.self_attn.kv_scale": Mock(),
+ }
+
+ name = "layers.0.self_attn.some_scale"
+ remapped = maybe_remap_kv_scale_name(name, params_dict)
+
+ assert remapped in params_dict or remapped == name or remapped is None
+
+
+def test_eagle3_lm_head_receives_quant_config():
+ """Eagle3LlamaForCausalLM must pass quant_config to ParallelLMHead.
+
+ Without quant_config, quantized lm_head weights (e.g. INT8 per-channel)
+ in Eagle3 drafter checkpoints fail to load because ParallelLMHead doesn't
+ expect weight_packed tensors.
+ """
+ from vllm.model_executor.models.llama_eagle3 import Eagle3LlamaForCausalLM
+
+ mock_quant_config = Mock()
+
+ mock_hf_config = Mock()
+ mock_hf_config.draft_vocab_size = 1000
+ mock_hf_config.hidden_size = 256
+ mock_hf_config.vocab_size = 32000
+ mock_hf_config.logit_scale = 1.0
+
+ mock_vllm_config = Mock()
+ mock_vllm_config.speculative_config.draft_model_config.hf_config = mock_hf_config
+ mock_vllm_config.model_config.get_num_layers.return_value = 32
+ mock_vllm_config.speculative_config.parallel_drafting = False
+
+ with (
+ patch("vllm.model_executor.models.llama_eagle3.LlamaModel") as MockModel,
+ patch("vllm.model_executor.models.llama_eagle3.ParallelLMHead") as MockLMHead,
+ patch("vllm.model_executor.models.llama_eagle3.LogitsProcessor"),
+ patch(
+ "vllm.model_executor.models.llama_eagle3.get_draft_quant_config",
+ return_value=mock_quant_config,
+ ),
+ ):
+ MockModel.return_value.use_aux_hidden_state = True
+
+ Eagle3LlamaForCausalLM(vllm_config=mock_vllm_config)
+
+ MockLMHead.assert_called_once()
+ call_kwargs = MockLMHead.call_args.kwargs
+ assert "quant_config" in call_kwargs, (
+ "ParallelLMHead must receive quant_config for quantized lm_head weights"
+ )
+ assert call_kwargs["quant_config"] is mock_quant_config, (
+ "ParallelLMHead must receive the draft model's quant_config"
+ )
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_enabled_custom_ops.py b/asian_ai/performance/vLLM/tests/model_executor/test_enabled_custom_ops.py
new file mode 100644
index 0000000000000000000000000000000000000000..1a1183ee46915fc98c4554b0d594676a4d08863f
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_enabled_custom_ops.py
@@ -0,0 +1,151 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+import torch
+
+from vllm._aiter_ops import rocm_aiter_ops
+from vllm.config import (
+ CompilationConfig,
+ VllmConfig,
+ get_cached_compilation_config,
+ set_current_vllm_config,
+)
+from vllm.model_executor.custom_op import CustomOp, op_registry
+from vllm.model_executor.layers.activation import (
+ GeluAndMul,
+ ReLUSquaredActivation,
+ SiluAndMul,
+)
+from vllm.model_executor.layers.fused_moe.router.fused_topk_router import (
+ dispatch_topk_sigmoid_func,
+ dispatch_topk_softmax_func,
+ vllm_topk_sigmoid,
+ vllm_topk_softmax,
+)
+from vllm.model_executor.layers.layernorm import RMSNorm
+from vllm.platforms import current_platform
+
+RMS_NORM_SUPPORTED_DTYPES = [torch.float16, torch.bfloat16]
+
+
+# Registered subclass for test
+@CustomOp.register("relu3")
+class Relu3(ReLUSquaredActivation):
+ pass
+
+
+@pytest.mark.parametrize(
+ "env, compilation_mode, backend, ops_enabled, default_on",
+ [
+ # Default values based on compile level
+ # - All by default (no Inductor compilation)
+ (None, 0, "eager", [True] * 4, True),
+ (None, 1, "eager", [True] * 4, True),
+ (None, 2, "eager", [True] * 4, True),
+ (None, 3, "eager", [True] * 4, True),
+ # - None by default (with Inductor)
+ (None, 0, "inductor", [True] * 4, True),
+ # - None by default (with Inductor)
+ (None, 1, "inductor", [False] * 4, False),
+ (None, 2, "inductor", [False] * 4, False),
+ (None, 3, "inductor", [False] * 4, False),
+ # Explicitly enabling/disabling
+ #
+ # Default: all
+ #
+ # All but SiluAndMul
+ ("+rms_norm,-silu_and_mul", 0, "inductor", [1, 0, 1, 1], True),
+ # Only ReLU3
+ ("none,-rms_norm,+relu3", 1, "eager", [0, 0, 0, 1], False),
+ # All but SiluAndMul
+ ("all,-silu_and_mul", 2, "inductor", [1, 0, 1, 1], True),
+ # All but ReLU3 (even if ReLU2 is on)
+ ("-relu3,+relu2", 3, "eager", [1, 1, 1, 0], True),
+ # RMSNorm and SiluAndMul
+ ("none,-relu3,+rms_norm,+silu_and_mul", 3, "eager", [1, 1, 0, 0], False),
+ # All but RMSNorm
+ ("-rms_norm", 3, "eager", [0, 1, 1, 1], True),
+ #
+ # Default: none
+ #
+ # Only ReLU3
+ ("none,+relu3", 3, "inductor", [0, 0, 0, 1], False),
+ # All but RMSNorm
+ ("all,-rms_norm", 3, "inductor", [0, 1, 1, 1], True),
+ ],
+)
+def test_enabled_ops(
+ env: str | None,
+ compilation_mode: int,
+ backend: str,
+ ops_enabled: list[int],
+ default_on: bool,
+):
+ custom_ops = env.split(",") if env else []
+ vllm_config = VllmConfig(
+ compilation_config=CompilationConfig(
+ backend=backend, mode=compilation_mode, custom_ops=custom_ops
+ )
+ )
+ get_cached_compilation_config.cache_clear()
+ with set_current_vllm_config(vllm_config):
+ assert CustomOp.default_on() == default_on
+
+ ops_enabled = [bool(x) for x in ops_enabled]
+
+ assert RMSNorm(1024).enabled() == ops_enabled[0]
+ assert op_registry["rms_norm"].enabled() == ops_enabled[0]
+
+ assert SiluAndMul().enabled() == ops_enabled[1]
+ assert op_registry["silu_and_mul"].enabled() == ops_enabled[1]
+
+ assert GeluAndMul().enabled() == ops_enabled[2]
+ assert op_registry["gelu_and_mul"].enabled() == ops_enabled[2]
+
+ # If registered, subclasses should follow their own name
+ assert Relu3().enabled() == ops_enabled[3]
+ assert op_registry["relu3"].enabled() == ops_enabled[3]
+
+ # Unregistered subclass
+ class SiluAndMul2(SiluAndMul):
+ pass
+
+ # Subclasses should not require registration
+ assert SiluAndMul2().enabled() == SiluAndMul().enabled()
+
+
+@pytest.mark.parametrize(
+ "env", ["all,none", "all,+rms_norm,all", "+rms_norm,-rms_norm"]
+)
+def test_enabled_ops_invalid(env: str):
+ with pytest.raises(Exception): # noqa
+ vllm_config = VllmConfig(
+ compilation_config=CompilationConfig(custom_ops=env.split(","))
+ )
+ with set_current_vllm_config(vllm_config):
+ RMSNorm(1024).enabled()
+
+
+@pytest.mark.parametrize(
+ "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
+)
+def test_topk_softmax_dispatch(use_rocm_aiter: bool):
+ topk_func = dispatch_topk_softmax_func(use_rocm_aiter)
+
+ if current_platform.is_rocm() and use_rocm_aiter:
+ assert topk_func == rocm_aiter_ops.topk_softmax
+ else:
+ assert topk_func == vllm_topk_softmax
+
+
+@pytest.mark.parametrize(
+ "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
+)
+def test_topk_sigmoid_dispatch(use_rocm_aiter: bool):
+ topk_func = dispatch_topk_sigmoid_func(use_rocm_aiter)
+
+ if current_platform.is_rocm() and use_rocm_aiter:
+ assert topk_func == rocm_aiter_ops.topk_sigmoid
+ else:
+ assert topk_func == vllm_topk_sigmoid
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_ernie45_vl_mrope.py b/asian_ai/performance/vLLM/tests/model_executor/test_ernie45_vl_mrope.py
new file mode 100644
index 0000000000000000000000000000000000000000..d68c56dd6a206afa8b294bb7696e0df3c1789a03
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_ernie45_vl_mrope.py
@@ -0,0 +1,143 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from dataclasses import dataclass
+
+import pytest
+import torch
+
+from vllm.model_executor.models.ernie45_vl import (
+ Ernie4_5_VLMoeForConditionalGeneration,
+)
+from vllm.multimodal.inputs import (
+ MultiModalFeatureSpec,
+ MultiModalFieldElem,
+ MultiModalKwargsItem,
+ PlaceholderRange,
+)
+
+pytestmark = pytest.mark.skip_global_cleanup
+
+
+@pytest.fixture(autouse=True, scope="module")
+def _force_cpu_default_device():
+ original = torch.get_default_device()
+ torch.set_default_device("cpu")
+ yield
+ torch.set_default_device(original)
+
+
+@dataclass
+class DummyConfig:
+ spatial_conv_size: int = 2
+ temporal_conv_size: int = 2
+
+
+def make_model(config: DummyConfig) -> Ernie4_5_VLMoeForConditionalGeneration:
+ model = object.__new__(Ernie4_5_VLMoeForConditionalGeneration)
+ model.config = config
+ return model
+
+
+def make_mm_feature(
+ *,
+ modality: str,
+ offset: int,
+ length: int,
+ grid_thw: tuple[int, int, int],
+) -> MultiModalFeatureSpec:
+ field_name = "image_grid_thw" if modality == "image" else "video_grid_thw"
+ return MultiModalFeatureSpec(
+ data=MultiModalKwargsItem(
+ {
+ field_name: MultiModalFieldElem(
+ data=torch.tensor(grid_thw),
+ field=None, # HACK.
+ ),
+ }
+ ),
+ modality=modality,
+ identifier="DUMMY",
+ mm_position=PlaceholderRange(offset=offset, length=length),
+ )
+
+
+def test_get_mrope_input_positions_text_only():
+ model = make_model(DummyConfig())
+
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=[11, 12, 13, 14, 15],
+ mm_features=[],
+ )
+
+ expected = torch.tensor(
+ [
+ [0, 1, 2, 3, 4],
+ [0, 1, 2, 3, 4],
+ [0, 1, 2, 3, 4],
+ ]
+ )
+
+ assert torch.equal(positions, expected)
+ assert delta == 0
+
+
+def test_get_mrope_input_positions_single_image():
+ model = make_model(DummyConfig())
+ mm_features = [
+ make_mm_feature(
+ modality="image",
+ offset=1,
+ length=4,
+ grid_thw=(1, 4, 4),
+ )
+ ]
+
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=[10, 20, 21, 22, 23, 30, 31],
+ mm_features=mm_features,
+ )
+
+ expected = torch.tensor(
+ [
+ [0, 1, 1, 1, 1, 3, 4],
+ [0, 1, 1, 2, 2, 3, 4],
+ [0, 1, 2, 1, 2, 3, 4],
+ ]
+ )
+
+ assert torch.equal(positions, expected)
+ assert delta == -2
+
+
+def test_get_mrope_input_positions_interleaved_image_and_video():
+ model = make_model(DummyConfig())
+ mm_features = [
+ make_mm_feature(
+ modality="image",
+ offset=1,
+ length=4,
+ grid_thw=(1, 4, 4),
+ ),
+ make_mm_feature(
+ modality="video",
+ offset=7,
+ length=2,
+ grid_thw=(2, 4, 2),
+ ),
+ ]
+
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=[10, 20, 21, 22, 23, 30, 31, 40, 41, 50, 51],
+ mm_features=mm_features,
+ )
+
+ expected = torch.tensor(
+ [
+ [0, 1, 1, 1, 1, 3, 4, 5, 5, 7, 8],
+ [0, 1, 1, 2, 2, 3, 4, 5, 6, 7, 8],
+ [0, 1, 2, 1, 2, 3, 4, 5, 5, 7, 8],
+ ]
+ )
+
+ assert torch.equal(positions, expected)
+ assert delta == -2
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_gemma_hidden_act.py b/asian_ai/performance/vLLM/tests/model_executor/test_gemma_hidden_act.py
new file mode 100644
index 0000000000000000000000000000000000000000..c7848e8e2a8548f728f406059817924504a7e05b
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_gemma_hidden_act.py
@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+import torch
+
+from vllm.model_executor.layers.activation import (
+ GeluAndMul,
+ SiluAndMul,
+ get_act_and_mul_fn,
+ get_act_fn,
+)
+from vllm.model_executor.models.gemma3 import Gemma3MLP
+from vllm.model_executor.models.gemma4 import Gemma4MLP
+
+
+@pytest.mark.parametrize(
+ ("activation_name", "expected_type"),
+ [
+ ("gelu_pytorch_tanh", GeluAndMul),
+ ("silu", SiluAndMul),
+ ("swish", SiluAndMul),
+ ],
+)
+def test_get_act_and_mul_fn_supports_gemma_hidden_act_aliases(
+ activation_name: str,
+ expected_type: type[torch.nn.Module],
+ default_vllm_config,
+) -> None:
+ assert isinstance(get_act_and_mul_fn(activation_name), expected_type)
+
+
+def test_get_act_fn_supports_swish_alias() -> None:
+ assert isinstance(get_act_fn("swish"), torch.nn.SiLU)
+
+
+@pytest.mark.parametrize("mlp_cls", [Gemma3MLP, Gemma4MLP])
+@pytest.mark.parametrize(
+ ("activation_name", "expected_type"),
+ [
+ ("gelu_pytorch_tanh", GeluAndMul),
+ ("silu", SiluAndMul),
+ ("swish", SiluAndMul),
+ ],
+)
+def test_gemma_mlp_supports_hidden_act_variants(
+ mlp_cls: type[torch.nn.Module],
+ activation_name: str,
+ expected_type: type[torch.nn.Module],
+ default_vllm_config,
+ dist_init,
+) -> None:
+ mlp = mlp_cls(
+ hidden_size=16,
+ intermediate_size=32,
+ hidden_activation=activation_name,
+ )
+
+ assert isinstance(mlp.act_fn, expected_type)
+ assert mlp(torch.randn(3, 16)).shape == (3, 16)
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_keye_mrope.py b/asian_ai/performance/vLLM/tests/model_executor/test_keye_mrope.py
new file mode 100644
index 0000000000000000000000000000000000000000..8aa6f0b5c11ea806f770d4995f2fb749ae7f1c57
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_keye_mrope.py
@@ -0,0 +1,145 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from dataclasses import dataclass, field
+
+import pytest
+import torch
+
+from vllm.model_executor.models.keye import KeyeForConditionalGeneration
+from vllm.multimodal.inputs import (
+ MultiModalFeatureSpec,
+ MultiModalFieldElem,
+ MultiModalKwargsItem,
+ PlaceholderRange,
+)
+
+pytestmark = pytest.mark.skip_global_cleanup
+
+
+@pytest.fixture(autouse=True, scope="module")
+def _force_cpu_default_device():
+ original = torch.get_default_device()
+ torch.set_default_device("cpu")
+ yield
+ torch.set_default_device(original)
+
+
+@dataclass
+class DummyVisionConfig:
+ spatial_merge_size: int = 2
+
+
+@dataclass
+class DummyConfig:
+ vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
+
+
+def make_model(config: DummyConfig) -> KeyeForConditionalGeneration:
+ model = object.__new__(KeyeForConditionalGeneration)
+ model.config = config
+ return model
+
+
+def make_mm_feature(
+ *,
+ modality: str,
+ offset: int,
+ length: int,
+ grid_thw: tuple[int, int, int] | list[tuple[int, int, int]],
+) -> MultiModalFeatureSpec:
+ field_name = "image_grid_thw" if modality == "image" else "video_grid_thw"
+ return MultiModalFeatureSpec(
+ data=MultiModalKwargsItem(
+ {
+ field_name: MultiModalFieldElem(
+ data=torch.tensor(grid_thw),
+ field=None, # HACK.
+ ),
+ }
+ ),
+ modality=modality,
+ identifier="DUMMY",
+ mm_position=PlaceholderRange(offset=offset, length=length),
+ )
+
+
+def test_get_mrope_input_positions_text_only():
+ model = make_model(DummyConfig())
+
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=[11, 12, 13, 14, 15],
+ mm_features=[],
+ )
+
+ expected = torch.tensor(
+ [
+ [0, 1, 2, 3, 4],
+ [0, 1, 2, 3, 4],
+ [0, 1, 2, 3, 4],
+ ]
+ )
+
+ assert torch.equal(positions, expected)
+ assert delta == 0
+
+
+def test_get_mrope_input_positions_single_image():
+ model = make_model(DummyConfig())
+ mm_features = [
+ make_mm_feature(
+ modality="image",
+ offset=1,
+ length=4,
+ grid_thw=(1, 4, 4),
+ )
+ ]
+
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=[10, 20, 21, 22, 23, 30, 31],
+ mm_features=mm_features,
+ )
+
+ expected = torch.tensor(
+ [
+ [0, 1, 1, 1, 1, 3, 4],
+ [0, 1, 1, 2, 2, 3, 4],
+ [0, 1, 2, 1, 2, 3, 4],
+ ]
+ )
+
+ assert torch.equal(positions, expected)
+ assert delta == -2
+
+
+def test_get_mrope_input_positions_interleaved_image_and_video():
+ model = make_model(DummyConfig())
+ mm_features = [
+ make_mm_feature(
+ modality="image",
+ offset=1,
+ length=4,
+ grid_thw=(1, 4, 4),
+ ),
+ make_mm_feature(
+ modality="video",
+ offset=7,
+ length=4,
+ grid_thw=[(2, 4, 2)],
+ ),
+ ]
+
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=[10, 20, 21, 22, 23, 30, 31, 40, 41, 42, 43, 50, 51],
+ mm_features=mm_features,
+ )
+
+ expected = torch.tensor(
+ [
+ [0, 1, 1, 1, 1, 3, 4, 5, 5, 7, 7, 9, 10],
+ [0, 1, 1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+ [0, 1, 2, 1, 2, 3, 4, 5, 5, 7, 7, 9, 10],
+ ]
+ )
+
+ assert torch.equal(positions, expected)
+ assert delta == -2
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_keye_vl1_5_mrope.py b/asian_ai/performance/vLLM/tests/model_executor/test_keye_vl1_5_mrope.py
new file mode 100644
index 0000000000000000000000000000000000000000..09739e557cfddf920e5605cd59ebd9cdafdc9767
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_keye_vl1_5_mrope.py
@@ -0,0 +1,145 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from dataclasses import dataclass, field
+
+import pytest
+import torch
+
+from vllm.model_executor.models.keye_vl1_5 import KeyeVL1_5ForConditionalGeneration
+from vllm.multimodal.inputs import (
+ MultiModalFeatureSpec,
+ MultiModalFieldElem,
+ MultiModalKwargsItem,
+ PlaceholderRange,
+)
+
+pytestmark = pytest.mark.skip_global_cleanup
+
+
+@pytest.fixture(autouse=True, scope="module")
+def _force_cpu_default_device():
+ original = torch.get_default_device()
+ torch.set_default_device("cpu")
+ yield
+ torch.set_default_device(original)
+
+
+@dataclass
+class DummyVisionConfig:
+ spatial_merge_size: int = 2
+
+
+@dataclass
+class DummyConfig:
+ vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
+
+
+def make_model(config: DummyConfig) -> KeyeVL1_5ForConditionalGeneration:
+ model = object.__new__(KeyeVL1_5ForConditionalGeneration)
+ model.config = config
+ return model
+
+
+def make_mm_feature(
+ *,
+ modality: str,
+ offset: int,
+ length: int,
+ grid_thw: tuple[int, int, int] | list[tuple[int, int, int]],
+ is_embed: list[bool] | None = None,
+) -> MultiModalFeatureSpec:
+ field_name = "image_grid_thw" if modality == "image" else "video_grid_thw"
+ return MultiModalFeatureSpec(
+ data=MultiModalKwargsItem(
+ {
+ field_name: MultiModalFieldElem(
+ data=torch.tensor(grid_thw),
+ field=None, # HACK.
+ ),
+ }
+ ),
+ modality=modality,
+ identifier="DUMMY",
+ mm_position=PlaceholderRange(
+ offset=offset,
+ length=length,
+ is_embed=None if is_embed is None else torch.tensor(is_embed),
+ ),
+ )
+
+
+def test_get_mrope_input_positions_text_only():
+ model = make_model(DummyConfig())
+
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=[11, 12, 13, 14, 15],
+ mm_features=[],
+ )
+
+ expected = torch.tensor(
+ [
+ [0, 1, 2, 3, 4],
+ [0, 1, 2, 3, 4],
+ [0, 1, 2, 3, 4],
+ ]
+ )
+
+ assert torch.equal(positions, expected)
+ assert delta == 0
+
+
+def test_get_mrope_input_positions_single_image():
+ model = make_model(DummyConfig())
+ mm_features = [
+ make_mm_feature(
+ modality="image",
+ offset=1,
+ length=4,
+ grid_thw=(1, 4, 4),
+ )
+ ]
+
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=[10, 20, 21, 22, 23, 30, 31],
+ mm_features=mm_features,
+ )
+
+ expected = torch.tensor(
+ [
+ [0, 1, 1, 1, 1, 3, 4],
+ [0, 1, 1, 2, 2, 3, 4],
+ [0, 1, 2, 1, 2, 3, 4],
+ ]
+ )
+
+ assert torch.equal(positions, expected)
+ assert delta == -2
+
+
+def test_get_mrope_input_positions_video_uses_embed_ranges():
+ model = make_model(DummyConfig())
+ mm_features = [
+ make_mm_feature(
+ modality="video",
+ offset=1,
+ length=8,
+ grid_thw=[(2, 4, 2)],
+ is_embed=[False, False, True, True, False, False, True, True],
+ )
+ ]
+
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=[10, 101, 102, 20, 21, 103, 104, 30, 31, 40, 41],
+ mm_features=mm_features,
+ )
+
+ expected = torch.tensor(
+ [
+ [0, 1, 2, 3, 3, 5, 6, 7, 7, 9, 10],
+ [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10],
+ [0, 1, 2, 3, 3, 5, 6, 7, 7, 9, 10],
+ ]
+ )
+
+ assert torch.equal(positions, expected)
+ assert delta == 0
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_mistral_large_3_eagle.py b/asian_ai/performance/vLLM/tests/model_executor/test_mistral_large_3_eagle.py
new file mode 100644
index 0000000000000000000000000000000000000000..486cd635cfdf3d16db8598ac073f91e6a2607984
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_mistral_large_3_eagle.py
@@ -0,0 +1,146 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+import torch.nn as nn
+
+from vllm.config.compilation import CompilationMode
+from vllm.model_executor.models import deepseek_v2 as deepseek_mod
+from vllm.model_executor.models import mistral_large_3_eagle as eagle_mod
+
+
+class DummyPPGroup:
+ world_size = 1
+ is_first_rank = True
+ is_last_rank = True
+
+
+class DummyEmbedding(nn.Module):
+ def __init__(self, vocab_size, hidden_size, *args, **kwargs):
+ super().__init__()
+ self.hidden_size = hidden_size
+
+ def forward(self, input_ids):
+ return torch.zeros(
+ (*input_ids.shape, self.hidden_size),
+ dtype=torch.float32,
+ device=input_ids.device,
+ )
+
+
+class DummyLinear(nn.Module):
+ def __init__(self, in_features, out_features, *args, **kwargs):
+ super().__init__()
+ self.out_features = out_features
+
+ def forward(self, x):
+ return torch.zeros(
+ (*x.shape[:-1], self.out_features),
+ dtype=x.dtype,
+ device=x.device,
+ )
+
+
+class DummyNorm(nn.Module):
+ def __init__(self, *args, **kwargs):
+ super().__init__()
+
+ def forward(self, hidden_states, residual=None):
+ return hidden_states, residual
+
+
+class DummyDecoderLayer(nn.Module):
+ def __init__(self, *args, **kwargs):
+ super().__init__()
+
+ def forward(self, positions, hidden_states, residual, llama_4_scaling=None):
+ return hidden_states, residual
+
+
+def make_vllm_config(
+ *, model_type="mistral3", qk_nope_head_dim=128, qk_rope_head_dim=64
+):
+ hf_config = SimpleNamespace(
+ model_type=model_type,
+ first_k_dense_replace=0,
+ vocab_size=32000,
+ hidden_size=16,
+ num_hidden_layers=1,
+ rms_norm_eps=1e-5,
+ qk_nope_head_dim=qk_nope_head_dim,
+ qk_rope_head_dim=qk_rope_head_dim,
+ )
+
+ return SimpleNamespace(
+ model_config=SimpleNamespace(hf_config=hf_config),
+ quant_config=None,
+ parallel_config=SimpleNamespace(
+ eplb_config=SimpleNamespace(num_redundant_experts=0),
+ ),
+ scheduler_config=SimpleNamespace(max_num_batched_tokens=8),
+ cache_config=None,
+ compilation_config=SimpleNamespace(mode=CompilationMode.NONE),
+ )
+
+
+@pytest.fixture(autouse=True)
+def patch_heavy_modules(monkeypatch):
+ monkeypatch.setattr(eagle_mod, "get_pp_group", lambda: DummyPPGroup())
+ monkeypatch.setattr(deepseek_mod, "get_pp_group", lambda: DummyPPGroup())
+
+ monkeypatch.setattr(eagle_mod, "VocabParallelEmbedding", DummyEmbedding)
+ monkeypatch.setattr(eagle_mod, "RowParallelLinear", DummyLinear)
+ monkeypatch.setattr(eagle_mod, "RMSNorm", DummyNorm)
+ monkeypatch.setattr(eagle_mod, "DeepseekV2DecoderLayer", DummyDecoderLayer)
+
+
+@pytest.mark.cpu_test
+@pytest.mark.parametrize(
+ ("model_type", "qk_nope_head_dim", "qk_rope_head_dim", "expected_use_mha"),
+ [
+ # MLA-style config: should not use MHA.
+ ("mistral3", 128, 64, False),
+ # No MLA dims: should use MHA, matching DeepseekV2Model.__init__ logic.
+ ("mistral3", 0, 0, True),
+ # DeepSeek model type always uses MHA by the parent logic.
+ ("deepseek", 128, 64, True),
+ ],
+)
+def test_eagle_mistral_large3_initializes_deepseek_runtime_attrs(
+ model_type,
+ qk_nope_head_dim,
+ qk_rope_head_dim,
+ expected_use_mha,
+):
+ vllm_config = make_vllm_config(
+ model_type=model_type,
+ qk_nope_head_dim=qk_nope_head_dim,
+ qk_rope_head_dim=qk_rope_head_dim,
+ )
+
+ model = eagle_mod.EagleMistralLarge3Model(vllm_config=vllm_config)
+
+ assert model.aux_hidden_state_layers == ()
+ assert model.use_mha is expected_use_mha
+
+ # Add this if your fix also copies num_redundant_experts from
+ # DeepseekV2Model.__init__.
+ assert model.num_redundant_experts == 0
+
+
+@pytest.mark.cpu_test
+def test_eagle_mistral_large3_forward_reuses_deepseek_parent_forward():
+ vllm_config = make_vllm_config()
+ model = eagle_mod.EagleMistralLarge3Model(vllm_config=vllm_config)
+
+ input_ids = torch.tensor([[1, 2, 3]])
+ positions = torch.tensor([[0, 1, 2]])
+ hidden_states = torch.zeros((1, 3, 16))
+
+ output = model(input_ids, positions, hidden_states)
+
+ assert isinstance(output, torch.Tensor)
+ assert output.shape == hidden_states.shape
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_model_load_with_params.py b/asian_ai/performance/vLLM/tests/model_executor/test_model_load_with_params.py
new file mode 100644
index 0000000000000000000000000000000000000000..e3649de4df2850d82cb5292159823b5298169c7e
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_model_load_with_params.py
@@ -0,0 +1,139 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import os
+
+import pytest
+
+from vllm.model_executor.layers.pooler import DispatchPooler
+from vllm.model_executor.layers.pooler.seqwise import CLSPool, MeanPool
+from vllm.model_executor.models.bert import BertEmbeddingModel
+from vllm.model_executor.models.roberta import RobertaEmbeddingModel
+from vllm.platforms import current_platform
+
+MAX_MODEL_LEN = 128
+MODEL_NAME = os.environ.get("MODEL_NAME", "BAAI/bge-base-en-v1.5")
+REVISION = os.environ.get("REVISION", "main")
+
+MODEL_NAME_ROBERTA = os.environ.get("MODEL_NAME", "intfloat/multilingual-e5-base")
+REVISION_ROBERTA = os.environ.get("REVISION", "main")
+
+
+@pytest.mark.skipif(
+ current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm."
+)
+def test_model_loading_with_params(vllm_runner, monkeypatch):
+ """
+ Test parameter weight loading with tp>1.
+ """
+ # to use apply_model
+ monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+ with vllm_runner(
+ model_name=MODEL_NAME,
+ revision=REVISION,
+ dtype="float16",
+ max_model_len=MAX_MODEL_LEN,
+ ) as vllm_model:
+ output = vllm_model.embed(
+ "Write a short story about a robot that dreams for the first time.\n"
+ )
+
+ model_config = vllm_model.llm.llm_engine.model_config
+ model_tokenizer = vllm_model.llm.llm_engine.tokenizer
+
+ # asserts on the bert model config file
+ assert model_config.encoder_config["max_seq_length"] == 512
+ assert model_config.encoder_config["do_lower_case"]
+
+ # asserts on the pooling config files
+ assert model_config.pooler_config.seq_pooling_type == "CLS"
+ assert model_config.pooler_config.tok_pooling_type == "ALL"
+ assert model_config.pooler_config.use_activation
+
+ # asserts on the tokenizer loaded
+ assert model_config.tokenizer == "BAAI/bge-base-en-v1.5"
+ assert model_tokenizer.model_max_length == 512
+
+ def check_model(model):
+ assert isinstance(model, BertEmbeddingModel)
+ assert isinstance(pooler := model.pooler, DispatchPooler)
+ assert isinstance(pooler.poolers_by_task["embed"].pooling, CLSPool)
+
+ vllm_model.apply_model(check_model)
+
+ assert output
+
+
+@pytest.mark.skipif(
+ current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm."
+)
+def test_roberta_model_loading_with_params(vllm_runner, monkeypatch):
+ """
+ Test parameter weight loading with tp>1.
+ """
+ # to use apply_model
+ monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+ with vllm_runner(
+ model_name=MODEL_NAME_ROBERTA,
+ revision=REVISION_ROBERTA,
+ dtype="float16",
+ max_model_len=MAX_MODEL_LEN,
+ ) as vllm_model:
+ output = vllm_model.embed(
+ "Write a short story about a robot that dreams for the first time.\n"
+ )
+
+ model_config = vllm_model.llm.llm_engine.model_config
+ model_tokenizer = vllm_model.llm.llm_engine.tokenizer
+
+ # asserts on the bert model config file
+ assert model_config.encoder_config["max_seq_length"] == 512
+ assert not model_config.encoder_config["do_lower_case"]
+
+ # asserts on the pooling config files
+ assert model_config.pooler_config.seq_pooling_type == "MEAN"
+ assert model_config.pooler_config.tok_pooling_type == "ALL"
+ assert model_config.pooler_config.use_activation
+
+ # asserts on the tokenizer loaded
+ assert model_config.tokenizer == "intfloat/multilingual-e5-base"
+ assert model_tokenizer.model_max_length == 512
+
+ def check_model(model):
+ assert isinstance(model, RobertaEmbeddingModel)
+ assert isinstance(pooler := model.pooler, DispatchPooler)
+ assert isinstance(pooler.poolers_by_task["embed"].pooling, MeanPool)
+
+ vllm_model.apply_model(check_model)
+
+ assert output
+
+
+@pytest.mark.skipif(
+ current_platform.is_rocm(), reason="Xformers backend is not supported on ROCm."
+)
+def test_facebook_roberta_model_loading_with_params(vllm_runner, monkeypatch):
+ """
+ Test loading roberta-base model with no lm_head.
+ """
+ # to use apply_model
+ monkeypatch.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+ model_name = "FacebookAI/roberta-base"
+ with vllm_runner(
+ model_name=model_name, dtype="float16", max_model_len=MAX_MODEL_LEN
+ ) as vllm_model:
+ output = vllm_model.embed(
+ "Write a short story about a robot that dreams for the first time.\n"
+ )
+
+ assert vllm_model.llm.llm_engine.model_config.tokenizer == model_name
+
+ def check_model(model):
+ assert isinstance(model, RobertaEmbeddingModel)
+ assert not hasattr(model, "lm_head")
+ assert isinstance(pooler := model.pooler, DispatchPooler)
+ assert isinstance(pooler.poolers_by_task["embed"].pooling, CLSPool)
+
+ vllm_model.apply_model(check_model)
+
+ assert output
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_nemotron_h_quantization.py b/asian_ai/performance/vLLM/tests/model_executor/test_nemotron_h_quantization.py
new file mode 100644
index 0000000000000000000000000000000000000000..b265a53e122eabd94d87947f9c62f8915a3af2ad
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_nemotron_h_quantization.py
@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from unittest.mock import Mock, patch
+
+
+def test_nemotron_h_lm_head_receives_quant_config():
+ from vllm.model_executor.models.nemotron_h import NemotronHForCausalLM
+
+ mock_quant_config = Mock()
+
+ mock_hf_config = Mock()
+ mock_hf_config.vocab_size = 128
+ mock_hf_config.hidden_size = 64
+
+ mock_vllm_config = Mock()
+ mock_vllm_config.model_config.hf_config = mock_hf_config
+ mock_vllm_config.model_config.dtype = None
+ mock_vllm_config.scheduler_config = Mock()
+ mock_vllm_config.quant_config = mock_quant_config
+
+ with (
+ patch("vllm.model_executor.models.nemotron_h.NemotronHModel") as MockModel,
+ patch("vllm.model_executor.models.nemotron_h.ParallelLMHead") as MockLMHead,
+ patch("vllm.model_executor.models.nemotron_h.LogitsProcessor"),
+ ):
+ MockModel.return_value.make_empty_intermediate_tensors = Mock()
+ MockModel.return_value.has_moe = False
+
+ NemotronHForCausalLM(vllm_config=mock_vllm_config)
+
+ MockLMHead.assert_called_once()
+ call_kwargs = MockLMHead.call_args.kwargs
+ assert call_kwargs["quant_config"] is mock_quant_config
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_oink_integration.py b/asian_ai/performance/vLLM/tests/model_executor/test_oink_integration.py
new file mode 100644
index 0000000000000000000000000000000000000000..5d9628147d68602418458af35ba68537619d5836
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_oink_integration.py
@@ -0,0 +1,111 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import multiprocessing
+import types
+
+import pytest
+
+from vllm.platforms import current_platform
+
+
+def _test_oink_availability_impl(
+ device_capability: tuple[int, int],
+ has_rmsnorm: bool,
+ has_fused_add_rms_norm: bool,
+ expected_available: bool,
+ expected_fused: bool,
+) -> None:
+ """Test OINK support detection with mocked state."""
+ import torch
+
+ from vllm import platforms
+
+ # Mock device capability (class method, override on class)
+ dc = platforms.interface.DeviceCapability(*device_capability)
+ platforms.current_platform.__class__.get_device_capability = lambda device_id=0: dc
+
+ # Mock oink ops
+ oink_ops = types.SimpleNamespace()
+ if has_rmsnorm:
+ oink_ops.rmsnorm = lambda x, w, eps: x
+ if has_fused_add_rms_norm:
+ oink_ops.fused_add_rms_norm = lambda x, residual, w, eps: None
+
+ torch.ops.oink = oink_ops
+
+ # Now import vllm modules with mocks in place (fresh import with mocked platform)
+ import vllm.kernels.oink_ops # noqa: F401
+ from vllm.ir.ops import fused_add_rms_norm, rms_norm
+
+ # Verify support checks
+ assert rms_norm.impls["oink"].supported is expected_available
+ assert fused_add_rms_norm.impls["oink"].supported is expected_fused
+
+
+@pytest.mark.parametrize(
+ "device_capability,has_rmsnorm,has_fused_add_rms_norm,expected_available,expected_fused",
+ [
+ # Case 1: < SM100, ops not supported
+ ((9, 0), True, False, False, False),
+ # Case 2: CUDA available and SM100, rmsnorm op registered
+ ((10, 0), True, False, True, False),
+ # Case 3: SM100 with both rmsnorm and fused_add_rms_norm
+ ((10, 0), True, True, True, True),
+ ],
+)
+@pytest.mark.skipif(not current_platform.is_cuda(), reason="Only test on CUDA")
+def test_oink_availability_checks(
+ device_capability: tuple[int, int],
+ has_rmsnorm: bool,
+ has_fused_add_rms_norm: bool,
+ expected_available: bool,
+ expected_fused: bool,
+):
+ """Test OINK support detection with clean import state for each parameter set."""
+
+ # Use spawn to run function in fresh process with clean imports
+ # TODO migrate to spawn utility:
+ # https://github.com/vllm-project/vllm/issues/41415
+ ctx = multiprocessing.get_context("spawn")
+ process = ctx.Process(
+ target=_test_oink_availability_impl,
+ args=(
+ device_capability,
+ has_rmsnorm,
+ has_fused_add_rms_norm,
+ expected_available,
+ expected_fused,
+ ),
+ )
+ process.start()
+ process.join()
+
+ if process.exitcode != 0:
+ raise AssertionError(
+ f"Subprocess test failed with exit code {process.exitcode}"
+ )
+
+
+def test_can_view_as_2d_stride_guard():
+ # No global import
+ import torch
+
+ # Import the helper from the kernels module.
+ from vllm.kernels.oink_ops import _can_view_as_2d
+
+ x = torch.zeros((2, 3, 4))
+ assert _can_view_as_2d(x) is True
+
+ # Size-1 dims should be ignored by the viewability check.
+ # Create a tensor where stride(0) != stride(1) * size(1) due to padding,
+ # but view(-1, H) is still valid because dim 1 has size 1.
+ base = torch.zeros((2, 10, 4))
+ x_singleton = base[:, :1, :]
+ x_singleton.view(-1, x_singleton.shape[-1])
+ assert _can_view_as_2d(x_singleton) is True
+
+ # Middle-dimension stride break: view(-1, hidden) should be invalid.
+ x2 = x[:, ::2, :]
+ with pytest.raises(RuntimeError):
+ x2.view(-1, x2.shape[-1])
+ assert _can_view_as_2d(x2) is False
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_paddleocr_vl_mrope.py b/asian_ai/performance/vLLM/tests/model_executor/test_paddleocr_vl_mrope.py
new file mode 100644
index 0000000000000000000000000000000000000000..2ab8ae93ced71d9096567baeeacf2e175d059479
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_paddleocr_vl_mrope.py
@@ -0,0 +1,210 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from dataclasses import dataclass, field
+
+import pytest
+import torch
+
+from vllm.model_executor.models.paddleocr_vl import (
+ PaddleOCRVLForConditionalGeneration,
+)
+from vllm.multimodal.inputs import (
+ MultiModalFeatureSpec,
+ MultiModalFieldElem,
+ MultiModalKwargsItem,
+ PlaceholderRange,
+)
+
+pytestmark = pytest.mark.skip_global_cleanup
+
+
+@pytest.fixture(autouse=True, scope="module")
+def _force_cpu_default_device():
+ original = torch.get_default_device()
+ torch.set_default_device("cpu")
+ yield
+ torch.set_default_device(original)
+
+
+@dataclass
+class DummyVisionConfig:
+ spatial_merge_size: int = 2
+ patch_size: int = 14
+
+
+@dataclass
+class DummyConfig:
+ image_token_id: int = 151655
+ video_token_id: int = 151654
+ vision_start_token_id: int = 151652
+ vision_end_token_id: int = 151653
+ vision_config: DummyVisionConfig = field(default_factory=DummyVisionConfig)
+
+
+def make_model(config: DummyConfig) -> PaddleOCRVLForConditionalGeneration:
+ model = object.__new__(PaddleOCRVLForConditionalGeneration)
+ model.config = config
+ return model
+
+
+def make_mm_feature(
+ *,
+ offset: int,
+ length: int,
+ image_grid_thw: tuple[int, int, int],
+) -> MultiModalFeatureSpec:
+ return MultiModalFeatureSpec(
+ data=MultiModalKwargsItem(
+ {
+ "image_grid_thw": MultiModalFieldElem(
+ data=torch.tensor(image_grid_thw),
+ field=None,
+ ),
+ }
+ ),
+ modality="image",
+ identifier="DUMMY",
+ mm_position=PlaceholderRange(offset=offset, length=length),
+ )
+
+
+def test_get_mrope_input_positions_text_only():
+ model = make_model(DummyConfig())
+ input_tokens = [11, 12, 13, 14, 15]
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=input_tokens,
+ mm_features=[],
+ )
+ expected = torch.tensor(
+ [
+ [0, 1, 2, 3, 4],
+ [0, 1, 2, 3, 4],
+ [0, 1, 2, 3, 4],
+ ]
+ )
+ assert torch.equal(positions, expected)
+ assert delta == 0
+
+
+def test_get_mrope_input_positions_single_image():
+ model = make_model(DummyConfig())
+ spatial_merge_size = model.config.vision_config.spatial_merge_size
+
+ t, h, w = 1, 2, 2
+ num_image_tokens = t * h * w
+
+ input_tokens = (
+ [10]
+ + [model.config.vision_start_token_id]
+ + [model.config.image_token_id] * num_image_tokens
+ + [model.config.vision_end_token_id]
+ + [30, 31]
+ )
+
+ mm_features = [
+ make_mm_feature(
+ offset=2, # 1 (text) + 1 (vision_start)
+ length=num_image_tokens,
+ image_grid_thw=(t, h * spatial_merge_size, w * spatial_merge_size),
+ )
+ ]
+
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=input_tokens,
+ mm_features=mm_features,
+ )
+
+ expected = torch.tensor(
+ [
+ [0, 1, 2, 2, 2, 2, 4, 5, 6],
+ [0, 1, 2, 2, 3, 3, 4, 5, 6],
+ [0, 1, 2, 3, 2, 3, 4, 5, 6],
+ ]
+ )
+
+ assert torch.equal(positions, expected)
+ expected_delta = (positions.max().item() + 1) - len(input_tokens)
+ assert delta == expected_delta
+
+
+def test_get_mrope_input_positions_multiple_images():
+ model = make_model(DummyConfig())
+ spatial_merge_size = model.config.vision_config.spatial_merge_size
+
+ t1, h1, w1 = 1, 2, 2
+ num1 = t1 * h1 * w1
+
+ t2, h2, w2 = 1, 1, 3
+ num2 = t2 * h2 * w2
+
+ input_tokens = (
+ [10]
+ + [model.config.vision_start_token_id]
+ + [model.config.image_token_id] * num1
+ + [model.config.vision_end_token_id]
+ + [20, 21]
+ + [model.config.vision_start_token_id]
+ + [model.config.image_token_id] * num2
+ + [model.config.vision_end_token_id]
+ + [30]
+ )
+
+ mm_features = [
+ make_mm_feature(
+ offset=2,
+ length=num1,
+ image_grid_thw=(t1, h1 * spatial_merge_size, w1 * spatial_merge_size),
+ ),
+ make_mm_feature(
+ offset=2 + num1 + 1 + 2 + 1,
+ length=num2,
+ image_grid_thw=(t2, h2 * spatial_merge_size, w2 * spatial_merge_size),
+ ),
+ ]
+
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=input_tokens,
+ mm_features=mm_features,
+ )
+
+ assert positions.shape == (3, 15)
+ assert not torch.equal(positions[:, 2:6], torch.arange(4).expand(3, 4) + 2)
+ assert not torch.equal(positions[:, 10:13], torch.arange(3).expand(3, 3) + 10)
+
+
+def test_get_mrope_input_positions_image_at_start():
+ model = make_model(DummyConfig())
+ spatial_merge_size = model.config.vision_config.spatial_merge_size
+
+ t, h, w = 1, 2, 2
+ num_tokens = t * h * w
+
+ input_tokens = (
+ [model.config.vision_start_token_id]
+ + [model.config.image_token_id] * num_tokens
+ + [model.config.vision_end_token_id]
+ + [10, 11]
+ )
+
+ mm_features = [
+ make_mm_feature(
+ offset=1, # start token at index 0
+ length=num_tokens,
+ image_grid_thw=(t, h * spatial_merge_size, w * spatial_merge_size),
+ )
+ ]
+
+ positions, delta = model.get_mrope_input_positions(
+ input_tokens=input_tokens,
+ mm_features=mm_features,
+ )
+
+ expected = torch.tensor(
+ [
+ [0, 1, 1, 1, 1, 3, 4, 5],
+ [0, 1, 1, 2, 2, 3, 4, 5],
+ [0, 1, 2, 1, 2, 3, 4, 5],
+ ]
+ )
+
+ assert torch.equal(positions, expected)
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_qwen3_5_quantization.py b/asian_ai/performance/vLLM/tests/model_executor/test_qwen3_5_quantization.py
new file mode 100644
index 0000000000000000000000000000000000000000..9c200476793d24670cce0c29ec4d388046556a11
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_qwen3_5_quantization.py
@@ -0,0 +1,78 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from unittest.mock import Mock, patch
+
+
+def test_qwen3_5_lm_head_receives_quant_config():
+ from vllm.model_executor.models.qwen3_5 import Qwen3_5ForCausalLMBase
+
+ mock_quant_config = Mock()
+
+ mock_hf_config = Mock()
+ mock_hf_config.tie_word_embeddings = False
+ mock_hf_config.vocab_size = 128
+ mock_hf_config.hidden_size = 64
+
+ mock_vllm_config = Mock()
+ mock_vllm_config.model_config.hf_text_config = mock_hf_config
+ mock_vllm_config.cache_config.mamba_cache_mode = "align"
+ mock_vllm_config.scheduler_config = Mock()
+ mock_vllm_config.quant_config = mock_quant_config
+ mock_vllm_config.lora_config = None
+
+ mock_pp_group = Mock()
+ mock_pp_group.is_last_rank = True
+
+ with (
+ patch("vllm.model_executor.models.qwen3_5.Qwen3_5Model") as MockModel,
+ patch("vllm.model_executor.models.qwen3_5.ParallelLMHead") as MockLMHead,
+ patch("vllm.model_executor.models.qwen3_5.LogitsProcessor"),
+ patch(
+ "vllm.model_executor.models.qwen3_5.get_pp_group",
+ return_value=mock_pp_group,
+ ),
+ ):
+ MockModel.return_value.make_empty_intermediate_tensors = Mock()
+
+ Qwen3_5ForCausalLMBase(vllm_config=mock_vllm_config)
+
+ MockLMHead.assert_called_once()
+ call_kwargs = MockLMHead.call_args.kwargs
+ assert call_kwargs["quant_config"] is mock_quant_config
+
+
+def test_qwen3_5_mtp_lm_head_receives_quant_config():
+ from vllm.config import CompilationMode
+ from vllm.model_executor.models.qwen3_5_mtp import Qwen3_5MTP
+
+ mock_quant_config = Mock()
+
+ mock_hf_config = Mock()
+ mock_hf_config.tie_word_embeddings = False
+ mock_hf_config.vocab_size = 128
+ mock_hf_config.hidden_size = 64
+
+ mock_vllm_config = Mock()
+ mock_vllm_config.model_config.hf_text_config = mock_hf_config
+ mock_vllm_config.cache_config.mamba_cache_mode = "align"
+ mock_vllm_config.compilation_config.mode = CompilationMode.NONE
+ mock_vllm_config.quant_config = mock_quant_config
+
+ mock_pp_group = Mock()
+ mock_pp_group.is_last_rank = True
+
+ with (
+ patch("vllm.model_executor.models.qwen3_5_mtp.Qwen3_5MultiTokenPredictor"),
+ patch("vllm.model_executor.models.qwen3_5_mtp.ParallelLMHead") as MockLMHead,
+ patch("vllm.model_executor.models.qwen3_5_mtp.LogitsProcessor"),
+ patch(
+ "vllm.model_executor.models.qwen3_5_mtp.get_pp_group",
+ return_value=mock_pp_group,
+ ),
+ ):
+ Qwen3_5MTP(vllm_config=mock_vllm_config)
+
+ MockLMHead.assert_called_once()
+ call_kwargs = MockLMHead.call_args.kwargs
+ assert call_kwargs["quant_config"] is mock_quant_config
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_qwen3_omni.py b/asian_ai/performance/vLLM/tests/model_executor/test_qwen3_omni.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc096ae0176317247cbcc7470b32087b434d0ece
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_qwen3_omni.py
@@ -0,0 +1,222 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from unittest.mock import Mock
+
+import pytest
+from transformers import PretrainedConfig
+
+from vllm.multimodal.processing import InputProcessingContext
+
+
+# Helper function to print input IDs with coalesced audio/video tokens.
+def print_input_ids(input_ids):
+ """
+ Print input IDs, compressing consecutive special tokens.
+ - 151675: <|audio_pad|>
+ - 151656: <|video_pad|>
+ """
+ if not input_ids:
+ print("[]")
+ return
+
+ result = []
+ i = 0
+
+ while i < len(input_ids):
+ current_id = input_ids[i]
+
+ # Check if it's a special token that should be compressed
+ if current_id in [151675, 151656]:
+ # Count consecutive occurrences
+ count = 1
+ while i + count < len(input_ids) and input_ids[i + count] == current_id:
+ count += 1
+
+ # Add compressed representation
+ token_name = "<|audio_pad|>" if current_id == 151675 else "<|video_pad|>"
+ result.append(f"{token_name} * {count}")
+ i += count
+ else:
+ # Regular token, just add it
+ result.append(str(current_id))
+ i += 1
+
+ print(", ".join(result))
+
+
+@pytest.fixture
+def mock_qwen3_omni_config():
+ """Create a mock Qwen3OmniMoeThinker config."""
+ config = Mock(spec=PretrainedConfig)
+ # Token IDs from https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
+ config.audio_token_id = 151675 # <|audio_pad|>
+ config.video_token_id = 151656 # <|video_pad|>
+ config.image_token_id = 151655 # <|image_pad|>
+ config.audio_start_token_id = 151669 # <|audio_start|>
+ config.audio_end_token_id = 151670 # <|audio_end|>
+ config.vision_start_token_id = 151652 # <|vision_start|>
+ config.position_id_per_seconds = 12.5
+
+ # Vision config
+ vision_config = Mock()
+ vision_config.spatial_merge_size = 2
+ config.vision_config = vision_config
+
+ return config
+
+
+@pytest.fixture
+def mock_processor():
+ """Create a mock HF processor."""
+ from transformers.models.whisper import WhisperFeatureExtractor
+
+ processor = Mock()
+ processor.audio_token = "<|audio_pad|>"
+ processor.image_token = "<|image_pad|>"
+ processor.video_token = "<|video_pad|>"
+
+ # Create a real WhisperFeatureExtractor instance for the feature_extractor attribute
+ feature_extractor = WhisperFeatureExtractor()
+ processor.feature_extractor = feature_extractor
+
+ return processor
+
+
+@pytest.fixture
+def mock_tokenizer():
+ """Create a mock tokenizer."""
+ tokenizer = Mock()
+ # Token IDs from https://huggingface.co/Qwen/Qwen3-Omni-30B-A3B-Instruct/blob/main/tokenizer_config.json
+ tokenizer.get_vocab = Mock(
+ return_value={
+ "<|audio_pad|>": 151675,
+ "<|video_pad|>": 151656,
+ "<|image_pad|>": 151655,
+ "<|audio_start|>": 151669,
+ "<|audio_end|>": 151670,
+ "<|vision_start|>": 151652,
+ "<|vision_end|>": 151653,
+ }
+ )
+ tokenizer.encode = Mock(
+ side_effect=lambda x: {
+ "<|vision_start|>": [151652],
+ "<|vision_end|>": [151653],
+ "<|audio_start|>": [151669],
+ "<|audio_end|>": [151670],
+ "<|audio_pad|>": [151675],
+ "<|image_pad|>": [151655],
+ "<|video_pad|>": [151656],
+ }.get(x, [0])
+ )
+ tokenizer.vision_bos_token = "<|vision_start|>"
+ tokenizer.vision_eos_token = "<|vision_end|>"
+ tokenizer.audio_bos_token = "<|audio_start|>"
+ tokenizer.audio_eos_token = "<|audio_end|>"
+ return tokenizer
+
+
+@pytest.fixture
+def mock_image_processor():
+ """Create a mock image processor."""
+ image_processor = Mock()
+ image_processor.merge_size = 2
+ return image_processor
+
+
+def test_qwen3_omni_get_updates_use_audio_in_video(
+ mock_qwen3_omni_config,
+ mock_processor,
+ mock_tokenizer,
+ mock_image_processor,
+):
+ """Test the get_updates_use_audio_in_video method directly."""
+
+ from vllm.model_executor.models.qwen3_omni_moe_thinker import (
+ Qwen3OmniMoeThinkerMultiModalProcessor,
+ Qwen3OmniMoeThinkerProcessingInfo,
+ )
+
+ # Create a mock context
+ mock_ctx = Mock(spec=InputProcessingContext)
+
+ # Create processing info
+ info = Qwen3OmniMoeThinkerProcessingInfo(mock_ctx)
+ info._get_expected_hidden_size = lambda: 100
+ info.get_hf_config = Mock(return_value=mock_qwen3_omni_config)
+ info.get_hf_processor = Mock(return_value=mock_processor)
+ info.get_tokenizer = Mock(return_value=mock_tokenizer)
+ info.get_image_processor = Mock(return_value=mock_image_processor)
+
+ # Create a mock dummy_inputs builder
+ mock_dummy_inputs = Mock()
+
+ # Create the processor
+ processor = Qwen3OmniMoeThinkerMultiModalProcessor(info, mock_dummy_inputs)
+
+ # Test parameters from reference video
+ # https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-Omni/demo/draw.mp4
+ audio_len = 85
+ video_grid_thw = [6, 36, 64]
+ video_second_per_grid_t = 2.0
+
+ # Call the method
+ updates = processor.get_updates_use_audio_in_video(
+ thinker_config=mock_qwen3_omni_config,
+ audio_len=audio_len,
+ video_grid_thw=video_grid_thw,
+ video_second_per_grid_t=video_second_per_grid_t,
+ )
+
+ # Updated input ids should align with HF implementation.
+ # 151669,
+ # <|video_pad|> * 576, <|audio_pad|> * 25,
+ # <|video_pad|> * 576, <|audio_pad|> * 25,
+ # <|video_pad|> * 576, <|audio_pad|> * 25,
+ # <|video_pad|> * 576, <|audio_pad|> * 10,
+ # <|video_pad|> * 1152,
+ # 151670
+ print_input_ids(updates)
+
+ # Verify structure
+ assert isinstance(updates, list)
+ assert len(updates) > 0
+
+ # Verify start and end tokens
+ audio_start_token_id = mock_qwen3_omni_config.audio_start_token_id
+ audio_end_token_id = mock_qwen3_omni_config.audio_end_token_id
+
+ assert updates[0] == audio_start_token_id
+ assert updates[-1] == audio_end_token_id
+
+ # Verify both audio and video tokens are present
+ audio_token_id = mock_qwen3_omni_config.audio_token_id
+ video_token_id = mock_qwen3_omni_config.video_token_id
+
+ audio_count = updates.count(audio_token_id)
+ video_count = updates.count(video_token_id)
+
+ assert audio_count == audio_len, (
+ f"Expected {audio_len} audio tokens, got {audio_count}"
+ )
+
+ # Calculate expected video token count
+ spatial_merge_size = mock_qwen3_omni_config.vision_config.spatial_merge_size
+ height = video_grid_thw[1] // spatial_merge_size
+ width = video_grid_thw[2] // spatial_merge_size
+ expected_video_count = video_grid_thw[0] * height * width
+
+ assert video_count == expected_video_count, (
+ f"Expected {expected_video_count} video tokens, got {video_count}"
+ )
+
+ # Total tokens should be: 1 (start) + audio_len + video_count + 1 (end)
+ expected_total = 1 + audio_len + expected_video_count + 1
+ assert len(updates) == expected_total, (
+ f"Expected {expected_total} total tokens, got {len(updates)}"
+ )
+
+
+if __name__ == "__main__":
+ pytest.main([__file__, "-v"])
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_qwen3_vl_mrope.py b/asian_ai/performance/vLLM/tests/model_executor/test_qwen3_vl_mrope.py
new file mode 100644
index 0000000000000000000000000000000000000000..897cbef4aa33d52185c7043b1d3a1784575cd478
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_qwen3_vl_mrope.py
@@ -0,0 +1,237 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import dataclasses
+import random
+from dataclasses import dataclass
+
+import pytest
+import torch
+
+from vllm.model_executor.models.qwen3_vl import Qwen3VLForConditionalGeneration
+from vllm.multimodal.inputs import (
+ MultiModalFeatureSpec,
+ MultiModalFieldElem,
+ MultiModalKwargsItem,
+ PlaceholderRange,
+)
+
+
+@pytest.fixture(autouse=True, scope="module")
+def _force_cpu_default_device():
+ # _get_mrope_input_positions returns CPU tensors (via torch.from_numpy).
+ # Ensure the default device is CPU so the rest of the test tensors match.
+ original = torch.get_default_device()
+ torch.set_default_device("cpu")
+ yield
+ torch.set_default_device(original)
+
+
+IMAGE_TOKEN_ID = 999
+VIDEO_TOKEN_ID = 888
+VISION_START_TOKEN_ID = 777
+VISION_END_TOKEN_ID = 778
+
+
+@dataclass
+class DummyVisionConfig:
+ spatial_merge_size: int = 1
+
+
+@dataclass
+class DummyConfig:
+ image_token_id: int = IMAGE_TOKEN_ID
+ video_token_id: int = VIDEO_TOKEN_ID
+ vision_start_token_id: int = VISION_START_TOKEN_ID
+ vision_end_token_id: int = VISION_END_TOKEN_ID
+ vision_config: DummyVisionConfig = dataclasses.field(
+ default_factory=DummyVisionConfig
+ )
+
+
+def make_video_embedding(
+ t, h, w, interleave_text_tokens: tuple[int, int], video_pruning_rate: float = 0.0
+):
+ """
+ Helper function to make a video embedding for a given video size and pruning rate.
+
+ Args:
+ t: Number of frames.
+ h: Number of rows.
+ w: Number of columns.
+ interleave_text_tokens: Tuple of minimum and maximum number of text tokens to
+ interleave with the video.
+ video_pruning_rate: Pruning rate for the video.
+
+ Returns:
+ Tuple of (unpruned_tokens_sequence, pruned_tokens_sequence, retention_mask)
+ """
+ unpruned_tokens_sequence = []
+ population = list(range(1, 100))
+
+ for _ in range(t):
+ num_prefix_tokens = random.randint(
+ interleave_text_tokens[0], interleave_text_tokens[1]
+ )
+
+ prefix_tokens = random.choices(population, k=num_prefix_tokens)
+ vision_tokens = (
+ [VISION_START_TOKEN_ID] + [VIDEO_TOKEN_ID] * h * w + [VISION_END_TOKEN_ID]
+ )
+
+ unpruned_tokens_sequence.extend(prefix_tokens)
+ unpruned_tokens_sequence.extend(vision_tokens)
+
+ unpruned_tokens_sequence = torch.tensor(unpruned_tokens_sequence, dtype=torch.long)
+ video_token_mask = unpruned_tokens_sequence == VIDEO_TOKEN_ID
+
+ pruning_mask = torch.bernoulli(video_token_mask.float() * video_pruning_rate).bool() # type: ignore[attr-defined]
+ # Sanity check that we don't prune what should not be pruned.
+ assert not pruning_mask[~video_token_mask].any()
+
+ retention_mask = ~pruning_mask
+ pruned_tokens_sequence = unpruned_tokens_sequence[retention_mask]
+ return unpruned_tokens_sequence, pruned_tokens_sequence, retention_mask
+
+
+@pytest.mark.parametrize("spatial_merge_size", [1, 2])
+@pytest.mark.parametrize("grid_thw", [[3, 8, 7], [128, 10, 12]])
+@pytest.mark.parametrize("num_prefix_tokens", [1, 11])
+@pytest.mark.parametrize("num_suffix_tokens", [0, 7])
+@pytest.mark.parametrize("video_pruning_rate", [0, 0.25, 0.75])
+@pytest.mark.parametrize("interleave_text_tokens", [(0, 0), (1, 4)])
+def test_match_qwen3vl_mrope_evs_on(
+ spatial_merge_size: int,
+ num_prefix_tokens: int,
+ grid_thw: tuple[int, int, int],
+ num_suffix_tokens: int,
+ video_pruning_rate: float,
+ interleave_text_tokens: tuple[int, int],
+):
+ hf_config = DummyConfig()
+ hf_config.vision_config.spatial_merge_size = spatial_merge_size
+
+ t, h, w = grid_thw
+ population = list(range(1, 100))
+ prefix_tokens = random.choices(population, k=num_prefix_tokens)
+ suffix_tokens = random.choices(population, k=num_suffix_tokens)
+
+ video_tokens, video_tokens_pruned, retention_mask = make_video_embedding(
+ t,
+ h // spatial_merge_size,
+ w // spatial_merge_size,
+ interleave_text_tokens=interleave_text_tokens,
+ video_pruning_rate=video_pruning_rate,
+ )
+ assert len(video_tokens) == len(retention_mask)
+
+ input_tokens = prefix_tokens + video_tokens.tolist() + suffix_tokens
+ input_tokens_pruned = prefix_tokens + video_tokens_pruned.tolist() + suffix_tokens
+
+ whole_sequence_retention_mask = torch.cat(
+ [
+ torch.ones(len(prefix_tokens), dtype=torch.bool),
+ retention_mask,
+ torch.ones(len(suffix_tokens), dtype=torch.bool),
+ ],
+ dim=0,
+ )
+
+ # Build the GT mrope for unpruned input.
+ mm_feature = MultiModalFeatureSpec(
+ data=MultiModalKwargsItem(
+ {
+ "video_grid_thw": MultiModalFieldElem(
+ data=torch.tensor(grid_thw),
+ field=None, # HACK.
+ ),
+ }
+ ),
+ modality="video",
+ identifier="DUMMY",
+ mm_position=PlaceholderRange(offset=0, length=len(input_tokens)),
+ )
+ expected_mrope, _ = Qwen3VLForConditionalGeneration._get_mrope_input_positions(
+ input_tokens=input_tokens,
+ mm_features=[mm_feature],
+ config=hf_config,
+ )
+
+ # Compute mrope for a video-only media (unpruned).
+ mm_feature = MultiModalFeatureSpec(
+ data=MultiModalKwargsItem(
+ {
+ "video_grid_thw": MultiModalFieldElem(
+ data=torch.tensor(grid_thw),
+ field=None, # HACK.
+ ),
+ }
+ ),
+ modality="video",
+ identifier="DUMMY",
+ mm_position=PlaceholderRange(offset=0, length=video_tokens.numel()),
+ )
+ video_mrope, _ = Qwen3VLForConditionalGeneration._get_mrope_input_positions(
+ input_tokens=video_tokens.tolist(),
+ mm_features=[mm_feature],
+ config=hf_config,
+ )
+ video_mrope = video_mrope.permute(1, 0) # [N, 3]
+ hidden_size = 16
+
+ is_video_embed = torch.isin(
+ video_tokens_pruned, torch.tensor([VIDEO_TOKEN_ID], dtype=torch.long)
+ )
+
+ expanded_positions = torch.full(
+ (len(video_tokens_pruned), 5),
+ fill_value=-100,
+ device=video_mrope.device,
+ dtype=torch.long,
+ )
+ expanded_positions[is_video_embed, :3] = video_mrope[retention_mask][is_video_embed]
+ expanded_positions[~is_video_embed, :3] = video_mrope[retention_mask][
+ ~is_video_embed
+ ]
+
+ is_vision_start = video_tokens_pruned == VISION_START_TOKEN_ID
+ expanded_positions[..., 3] = is_vision_start
+ expanded_positions[..., 4] = is_video_embed
+
+ # Check that all positions were filled, since we initialized them as negative.
+ assert (expanded_positions >= 0).all()
+
+ video_embeddings = torch.empty(
+ (len(video_tokens_pruned), hidden_size), device=video_mrope.device
+ )
+
+ video_embeddings = torch.cat(
+ [
+ video_embeddings,
+ expanded_positions.float(),
+ ],
+ dim=1,
+ )
+ multimodal_embeddings = [video_embeddings]
+
+ expected_mrope_masked = expected_mrope[:, whole_sequence_retention_mask]
+
+ # Initialize computed_mrope with sequential positions for all prefix tokens
+ computed_mrope = torch.empty((3, len(input_tokens_pruned)), dtype=torch.long)
+ computed_mrope[:, 0 : len(prefix_tokens)] = expected_mrope[
+ :, 0 : len(prefix_tokens)
+ ]
+
+ # Paranoia check that computed_mrope is wrong.
+ assert not torch.equal(computed_mrope, expected_mrope_masked)
+
+ _, actual_mrope, _ = Qwen3VLForConditionalGeneration._recompute_mrope_positions(
+ input_ids=input_tokens_pruned,
+ multimodal_embeddings=multimodal_embeddings,
+ mrope_positions=computed_mrope,
+ num_computed_tokens=len(prefix_tokens),
+ vision_start_token_id=hf_config.vision_start_token_id,
+ image_token_id=hf_config.image_token_id,
+ video_token_id=hf_config.video_token_id,
+ )
+
+ assert torch.equal(actual_mrope, expected_mrope_masked)
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_routed_experts_capture.py b/asian_ai/performance/vLLM/tests/model_executor/test_routed_experts_capture.py
new file mode 100644
index 0000000000000000000000000000000000000000..afa1b2da15b187f721b25bf5eaa23ab01bb01d7d
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_routed_experts_capture.py
@@ -0,0 +1,252 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import types
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+import torch
+
+from vllm.distributed.eplb.eplb_state import EplbLayerState
+from vllm.model_executor.layers.fused_moe.config import RoutingMethodType
+from vllm.model_executor.layers.fused_moe.routed_experts_capturer import (
+ RoutedExpertsCapturer,
+)
+from vllm.model_executor.layers.fused_moe.router.base_router import BaseRouter
+
+pytestmark = pytest.mark.cpu_test
+
+_REC_MODULE = "vllm.model_executor.layers.fused_moe.routed_experts_capturer"
+
+
+def _capturer_with_buffer(
+ *,
+ max_tokens: int = 8,
+ num_layers: int = 4,
+ num_experts_per_tok: int = 2,
+ dp_rank: int = 0,
+ tp_size: int = 1,
+) -> RoutedExpertsCapturer:
+ # Bypass __init__ so the test can use a CPU buffer and skip the
+ # VllmConfig dependency. The CUDA device-tensor allocation in the
+ # real constructor is not what we are exercising here.
+ c = RoutedExpertsCapturer.__new__(RoutedExpertsCapturer)
+ c.dp_rank = dp_rank
+ c.tp_size = tp_size
+ c.device_buffer = torch.full(
+ (max_tokens, num_layers, num_experts_per_tok),
+ -1,
+ dtype=torch.int32,
+ )
+ return c
+
+
+class DummyRouter(BaseRouter):
+ @property
+ def routing_method_type(self) -> RoutingMethodType:
+ return RoutingMethodType.FUSED_TOPK
+
+ def _compute_routing(
+ self, hidden_states, router_logits, indices_type, *, input_ids=None
+ ):
+ topk_ids = torch.tensor([[1, 2], [3, 4]], dtype=torch.int64)
+ topk_weights = torch.ones_like(topk_ids, dtype=torch.float32)
+ return topk_weights, topk_ids
+
+ def _apply_eplb_mapping(self, topk_ids: torch.Tensor) -> torch.Tensor:
+ # Make mapping observable without requiring CUDA EPLB path.
+ return topk_ids + 10
+
+
+def _make_router(eplb_state: EplbLayerState | None = None) -> DummyRouter:
+ return DummyRouter(
+ top_k=2,
+ global_num_experts=16,
+ eplb_state=eplb_state,
+ )
+
+
+def test_base_router_capture_pre_eplb_mapping():
+ router = _make_router()
+ captured = []
+
+ def capture_fn(ids):
+ captured.append(ids.clone())
+
+ router.set_capture_fn(capture_fn)
+ topk_weights, topk_ids = router.select_experts(
+ hidden_states=torch.empty(1),
+ router_logits=torch.empty(1),
+ )
+
+ assert topk_weights.shape == topk_ids.shape
+ assert len(captured) == 1
+ assert torch.equal(captured[0], torch.tensor([[1, 2], [3, 4]]))
+ assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
+
+
+def test_base_router_capture_with_eplb_enabled():
+ eplb_state = EplbLayerState()
+ eplb_state.expert_load_view = torch.zeros(32, dtype=torch.int64)
+ eplb_state.logical_to_physical_map = torch.arange(32).view(32, 1)
+ eplb_state.logical_replica_count = torch.ones(32, dtype=torch.int64)
+ eplb_state.should_record_tensor = torch.ones((), dtype=torch.bool)
+ eplb_state.num_unpadded_tokens_tensors = [torch.tensor(0, dtype=torch.int32)]
+ router = _make_router(eplb_state=eplb_state)
+
+ captured = []
+
+ def capture_fn(ids):
+ captured.append(ids.clone())
+
+ router.set_capture_fn(capture_fn)
+ _, topk_ids = router.select_experts(
+ hidden_states=torch.empty(1),
+ router_logits=torch.empty(1),
+ )
+
+ assert len(captured) == 1
+ # Capture should see logical ids pre-EPLB mapping.
+ assert torch.equal(captured[0], torch.tensor([[1, 2], [3, 4]]))
+ # Our DummyRouter mapping adds +10.
+ assert torch.equal(topk_ids, torch.tensor([[11, 12], [13, 14]]))
+
+
+def test_gpu_model_runner_binds_router_capture(monkeypatch):
+ from vllm.v1.worker import gpu_model_runner as gmr
+
+ class _DummyRouter:
+ _routing_replay_out: torch.Tensor | None = None
+
+ class DummyFusedMoE:
+ def __init__(self):
+ self.layer_id = 7
+ self.router = _make_router()
+
+ class DummyCapturer:
+ def __init__(self):
+ self.calls = []
+
+ def capture(self, layer_id, topk_ids):
+ self.calls.append((layer_id, topk_ids))
+
+ dummy_module = DummyFusedMoE()
+
+ # Patch the runtime import inside _bind_routed_experts_capturer.
+ import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
+
+ monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
+
+ dummy_self = types.SimpleNamespace(
+ compilation_config=types.SimpleNamespace(
+ static_forward_context={"dummy": dummy_module}
+ )
+ )
+
+ capturer = DummyCapturer()
+ gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, capturer)
+
+ assert dummy_module.router.capture_fn is not None
+ dummy_module.router.capture_fn(torch.tensor([[5, 6]]))
+
+ assert len(capturer.calls) == 1
+ layer_id, topk_ids = capturer.calls[0]
+ assert layer_id == 7
+ assert torch.equal(topk_ids, torch.tensor([[5, 6]]))
+
+
+def test_gpu_model_runner_binding_stage(monkeypatch):
+ from vllm.v1.worker import gpu_model_runner as gmr
+
+ class DummyFusedMoE:
+ def __init__(self):
+ self.layer_id = 11
+ self.router = _make_router()
+
+ class DummyCapturer:
+ def __init__(self):
+ self.calls = []
+
+ def capture(self, layer_id, topk_ids):
+ self.calls.append((layer_id, topk_ids))
+
+ dummy_module = DummyFusedMoE()
+
+ import vllm.model_executor.layers.fused_moe.layer as fused_moe_layer
+
+ monkeypatch.setattr(fused_moe_layer, "MoERunner", DummyFusedMoE)
+
+ dummy_self = types.SimpleNamespace(
+ compilation_config=types.SimpleNamespace(
+ static_forward_context={"dummy": dummy_module}
+ )
+ )
+
+ # Before binding, no capture hook.
+ assert dummy_module.router.capture_fn is None
+
+ capturer = DummyCapturer()
+ gmr.GPUModelRunner._bind_routed_experts_capturer(dummy_self, capturer)
+
+ # After binding, hook should exist and be callable.
+ assert callable(dummy_module.router.capture_fn)
+ dummy_module.router.capture_fn(torch.tensor([[9, 10]]))
+ assert len(capturer.calls) == 1
+
+
+def test_routed_experts_capturer_single_dp_no_metadata():
+ """dp_metadata is None: capture writes the full topk_ids rows."""
+ capturer = _capturer_with_buffer(dp_rank=0)
+ topk = torch.tensor([[1, 2], [3, 4], [5, 6]], dtype=torch.int32)
+ ctx = SimpleNamespace(dp_metadata=None)
+ with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
+ capturer.capture(layer_id=0, topk_ids=topk)
+ assert torch.equal(capturer.device_buffer[:3, 0, :], topk)
+ assert capturer.device_buffer[3, 0, 0].item() == -1
+
+
+def test_routed_experts_capturer_dp_naive_concatenated_all_ranks():
+ """n == sum(num_tokens_dp): slice this rank's segment from concatenated topk."""
+ capturer = _capturer_with_buffer(dp_rank=1)
+ num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
+ ctx = SimpleNamespace(
+ dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
+ )
+ # Concatenated order: rank0 rows then rank1 rows.
+ topk = torch.tensor(
+ [[0, 1], [2, 3], [10, 11], [12, 13], [14, 15]], dtype=torch.int32
+ )
+ with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
+ capturer.capture(layer_id=0, topk_ids=topk)
+ want = topk[2:5]
+ assert torch.equal(capturer.device_buffer[:3, 0, :], want)
+
+
+def test_routed_experts_capturer_dp_modular_local_tokens():
+ """n == token_num_per_dp: topk is already local to this DP rank."""
+ capturer = _capturer_with_buffer(dp_rank=1)
+ num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
+ ctx = SimpleNamespace(
+ dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
+ )
+ topk = torch.tensor([[10, 11], [12, 13], [14, 15]], dtype=torch.int32)
+ with patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx):
+ capturer.capture(layer_id=0, topk_ids=topk)
+ assert torch.equal(capturer.device_buffer[:3, 0, :], topk)
+
+
+def test_routed_experts_capturer_dp_unexpected_batch_raises():
+ """Mismatch between topk batch dim and DP layout: fail fast."""
+ capturer = _capturer_with_buffer(dp_rank=0)
+ num_tokens_dp = torch.tensor([2, 3], dtype=torch.int32)
+ ctx = SimpleNamespace(
+ dp_metadata=SimpleNamespace(num_tokens_across_dp_cpu=num_tokens_dp)
+ )
+ # total=5, local=2: n=1 matches neither naive (5) nor modular (2).
+ topk = torch.tensor([[1, 2]], dtype=torch.int32)
+ with (
+ patch(f"{_REC_MODULE}.get_forward_context", return_value=ctx),
+ pytest.raises(AssertionError, match="unexpected topk_ids batch dim"),
+ ):
+ capturer.capture(layer_id=0, topk_ids=topk)
+ assert capturer.device_buffer[0, 0, 0].item() == -1
diff --git a/asian_ai/performance/vLLM/tests/model_executor/test_weight_utils.py b/asian_ai/performance/vLLM/tests/model_executor/test_weight_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..c2894f82ed6d39e536fd999bfdf550127481f386
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/model_executor/test_weight_utils.py
@@ -0,0 +1,285 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import tempfile
+
+import huggingface_hub.constants
+import pytest
+from huggingface_hub.utils import LocalEntryNotFoundError
+
+from vllm.model_executor.model_loader.weight_utils import (
+ download_weights_from_hf,
+ maybe_remap_kv_scale_name,
+)
+
+
+def test_download_weights_from_hf():
+ with tempfile.TemporaryDirectory() as tmpdir:
+ # assert LocalEntryNotFoundError error is thrown
+ # if offline is set and model is not cached
+ huggingface_hub.constants.HF_HUB_OFFLINE = True
+ with pytest.raises(LocalEntryNotFoundError):
+ download_weights_from_hf(
+ "facebook/opt-125m",
+ allow_patterns=["*.safetensors", "*.bin"],
+ cache_dir=tmpdir,
+ )
+
+ # download the model
+ huggingface_hub.constants.HF_HUB_OFFLINE = False
+ download_weights_from_hf(
+ "facebook/opt-125m",
+ allow_patterns=["*.safetensors", "*.bin"],
+ cache_dir=tmpdir,
+ )
+
+ # now it should work offline
+ huggingface_hub.constants.HF_HUB_OFFLINE = True
+ assert (
+ download_weights_from_hf(
+ "facebook/opt-125m",
+ allow_patterns=["*.safetensors", "*.bin"],
+ cache_dir=tmpdir,
+ )
+ is not None
+ )
+
+
+class TestMaybeRemapKvScaleName:
+ """Tests for maybe_remap_kv_scale_name covering all checkpoint formats."""
+
+ PARAMS_DICT = {
+ "model.layers.0.self_attn.attn.k_scale": None,
+ "model.layers.0.self_attn.attn.v_scale": None,
+ "model.layers.0.self_attn.attn.q_scale": None,
+ "model.layers.0.self_attn.qkv_proj.weight": None,
+ }
+
+ def test_qkv_proj_k_scale(self):
+ """Qwen3-MoE / llm-compressor format: qkv_proj.k_scale -> attn.k_scale
+ Regression test for https://github.com/vllm-project/vllm/issues/25047"""
+ result = maybe_remap_kv_scale_name(
+ "model.layers.0.self_attn.qkv_proj.k_scale", self.PARAMS_DICT
+ )
+ assert result == "model.layers.0.self_attn.attn.k_scale"
+
+ def test_qkv_proj_v_scale(self):
+ """Qwen3-MoE / llm-compressor format: qkv_proj.v_scale -> attn.v_scale
+ Regression test for https://github.com/vllm-project/vllm/issues/25047"""
+ result = maybe_remap_kv_scale_name(
+ "model.layers.0.self_attn.qkv_proj.v_scale", self.PARAMS_DICT
+ )
+ assert result == "model.layers.0.self_attn.attn.v_scale"
+
+ def test_modelopt_k_proj_k_scale(self):
+ """ModelOpt format: k_proj.k_scale -> attn.k_scale"""
+ result = maybe_remap_kv_scale_name(
+ "model.layers.0.self_attn.k_proj.k_scale", self.PARAMS_DICT
+ )
+ assert result == "model.layers.0.self_attn.attn.k_scale"
+
+ def test_modelopt_v_proj_v_scale(self):
+ """ModelOpt format: v_proj.v_scale -> attn.v_scale"""
+ result = maybe_remap_kv_scale_name(
+ "model.layers.0.self_attn.v_proj.v_scale", self.PARAMS_DICT
+ )
+ assert result == "model.layers.0.self_attn.attn.v_scale"
+
+ def test_deprecated_kv_scale(self):
+ """Old format: kv_scale -> attn.k_scale (deprecated)"""
+ result = maybe_remap_kv_scale_name(
+ "model.layers.0.self_attn.kv_scale", self.PARAMS_DICT
+ )
+ assert result == "model.layers.0.self_attn.attn.k_scale"
+
+ def test_default_bare_k_scale(self):
+ """Default format: .k_scale -> .attn.k_scale"""
+ result = maybe_remap_kv_scale_name(
+ "model.layers.0.self_attn.k_scale", self.PARAMS_DICT
+ )
+ assert result == "model.layers.0.self_attn.attn.k_scale"
+
+ def test_non_scale_name_unchanged(self):
+ """Non-scale names should be returned unchanged."""
+ name = "model.layers.0.self_attn.qkv_proj.weight"
+ result = maybe_remap_kv_scale_name(name, self.PARAMS_DICT)
+ assert result == name
+
+ def test_nvfp4_modelopt_k_proj_k_scale(self):
+ """ModelOpt NVFP4 format (e.g. nvidia/Qwen3-30B-A3B-NVFP4):
+ k_proj.k_scale -> attn.k_scale.
+ Validates that NVFP4 checkpoints are not broken by this change."""
+ result = maybe_remap_kv_scale_name(
+ "model.layers.0.self_attn.k_proj.k_scale", self.PARAMS_DICT
+ )
+ assert result == "model.layers.0.self_attn.attn.k_scale"
+
+ def test_nvfp4_modelopt_v_proj_v_scale(self):
+ """ModelOpt NVFP4 format (e.g. nvidia/Qwen3-30B-A3B-NVFP4):
+ v_proj.v_scale -> attn.v_scale.
+ Validates that NVFP4 checkpoints are not broken by this change."""
+ result = maybe_remap_kv_scale_name(
+ "model.layers.0.self_attn.v_proj.v_scale", self.PARAMS_DICT
+ )
+ assert result == "model.layers.0.self_attn.attn.v_scale"
+
+ def test_qwen3_vl_moe_qkv_proj_k_scale(self):
+ """Qwen3-VL-MoE uses the same fused qkv_proj naming as Qwen3-MoE.
+ Regression test for qwen3_vl_moe.py fix (same bug as #25047)."""
+ result = maybe_remap_kv_scale_name(
+ "model.layers.0.self_attn.qkv_proj.k_scale", self.PARAMS_DICT
+ )
+ assert result == "model.layers.0.self_attn.attn.k_scale"
+
+ def test_qwen3_vl_moe_qkv_proj_v_scale(self):
+ """Qwen3-VL-MoE uses the same fused qkv_proj naming as Qwen3-MoE.
+ Regression test for qwen3_vl_moe.py fix (same bug as #25047)."""
+ result = maybe_remap_kv_scale_name(
+ "model.layers.0.self_attn.qkv_proj.v_scale", self.PARAMS_DICT
+ )
+ assert result == "model.layers.0.self_attn.attn.v_scale"
+
+ def test_nvfp4_weight_scale_not_remapped(self):
+ """NVFP4 weight_scale should not be touched by remap (not a kv scale)."""
+ name = "model.layers.0.self_attn.k_proj.weight_scale"
+ result = maybe_remap_kv_scale_name(name, self.PARAMS_DICT)
+ assert result == name
+
+ def test_nvfp4_input_scale_not_remapped(self):
+ """NVFP4 input_scale should not be touched by remap (not a kv scale)."""
+ name = "model.layers.0.self_attn.k_proj.input_scale"
+ result = maybe_remap_kv_scale_name(name, self.PARAMS_DICT)
+ assert result == name
+
+ def test_missing_target_returns_none(self):
+ """If remapped name not in params_dict, return None."""
+ empty_params: dict[str, None] = {}
+ result = maybe_remap_kv_scale_name(
+ "model.layers.0.self_attn.qkv_proj.k_scale", empty_params
+ )
+ assert result is None
+
+
+class TestKvCacheScaleMapper:
+ """The `WeightsMapper` returned by `get_cache_scale_mapper` replaces the
+ per-model `maybe_remap_kv_scale_name` calls. It must remap the same set of
+ checkpoint formats (the non-`params_dict`-dependent ones) and be idempotent
+ so it composes safely with a model's own qkv/gate_up `hf_to_vllm_mapper`."""
+
+ def _mapper(self):
+ # `get_cache_scale_mapper` does not use `self`; call it on the base
+ # class to get the default (non-config-specific) mapper.
+ from vllm.model_executor.layers.quantization.base_config import (
+ QuantizationConfig,
+ )
+
+ return QuantizationConfig.get_cache_scale_mapper()
+
+ def _map(self, name: str) -> str | None:
+ return self._mapper()._map_name(name)
+
+ @pytest.mark.parametrize(
+ "name,expected",
+ [
+ # Qwen3-MoE / llm-compressor fused qkv_proj
+ (
+ "model.layers.0.self_attn.qkv_proj.k_scale",
+ "model.layers.0.self_attn.attn.k_scale",
+ ),
+ (
+ "model.layers.0.self_attn.qkv_proj.v_scale",
+ "model.layers.0.self_attn.attn.v_scale",
+ ),
+ # ModelOpt / NVFP4 k_proj/v_proj
+ (
+ "model.layers.0.self_attn.k_proj.k_scale",
+ "model.layers.0.self_attn.attn.k_scale",
+ ),
+ (
+ "model.layers.0.self_attn.v_proj.v_scale",
+ "model.layers.0.self_attn.attn.v_scale",
+ ),
+ # deprecated fused kv_scale and bare scales
+ (
+ "model.layers.0.self_attn.kv_scale",
+ "model.layers.0.self_attn.attn.k_scale",
+ ),
+ (
+ "model.layers.0.self_attn.k_scale",
+ "model.layers.0.self_attn.attn.k_scale",
+ ),
+ # NemotronH mixer
+ (
+ "model.layers.0.mixer.k_proj.k_scale",
+ "model.layers.0.mixer.attn.k_scale",
+ ),
+ # already in vLLM form -> unchanged (idempotent)
+ (
+ "model.layers.0.self_attn.attn.k_scale",
+ "model.layers.0.self_attn.attn.k_scale",
+ ),
+ # non-kv scales must not be touched
+ (
+ "model.layers.0.self_attn.k_proj.weight_scale",
+ "model.layers.0.self_attn.k_proj.weight_scale",
+ ),
+ (
+ "model.layers.0.self_attn.k_proj.input_scale",
+ "model.layers.0.self_attn.k_proj.input_scale",
+ ),
+ # regular weights untouched
+ (
+ "model.layers.0.self_attn.q_proj.weight",
+ "model.layers.0.self_attn.q_proj.weight",
+ ),
+ ],
+ )
+ def test_remap(self, name, expected):
+ assert self._map(name) == expected
+
+ @pytest.mark.parametrize(
+ "name",
+ [
+ "model.layers.0.self_attn.k_scale",
+ "model.layers.0.self_attn.k_proj.k_scale",
+ "model.layers.0.self_attn.qkv_proj.v_scale",
+ "model.layers.0.mixer.k_proj.k_scale",
+ ],
+ )
+ def test_idempotent(self, name):
+ once = self._map(name)
+ assert once is not None
+ assert self._map(once) == once
+
+ def test_composes_with_qkv_mapper(self):
+ """Applied together with a model's qkv/gate_up mapper, the regex scale
+ rules run before the substr rename, so scales are normalized to `.attn.`
+ and regular projections are still fused correctly."""
+ from vllm.model_executor.models.utils import WeightsMapper
+
+ model_mapper = WeightsMapper(
+ orig_to_new_substr={
+ ".q_proj": ".qkv_proj.q",
+ ".k_proj": ".qkv_proj.k",
+ ".v_proj": ".qkv_proj.v",
+ }
+ )
+ # AutoWeightsLoader does `mapper |= cache_scale_mapper`
+ combined = model_mapper | self._mapper()
+
+ assert (
+ combined._map_name("model.layers.0.self_attn.q_proj.weight")
+ == "model.layers.0.self_attn.qkv_proj.q.weight"
+ )
+ assert (
+ combined._map_name("model.layers.0.self_attn.k_proj.k_scale")
+ == "model.layers.0.self_attn.attn.k_scale"
+ )
+ assert (
+ combined._map_name("model.layers.0.self_attn.k_scale")
+ == "model.layers.0.self_attn.attn.k_scale"
+ )
+
+
+if __name__ == "__main__":
+ test_download_weights_from_hf()
diff --git a/asian_ai/performance/vLLM/tests/models/__init__.py b/asian_ai/performance/vLLM/tests/models/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/models/fixtures/audioflamingo3/expected_results_batched.json b/asian_ai/performance/vLLM/tests/models/fixtures/audioflamingo3/expected_results_batched.json
new file mode 100644
index 0000000000000000000000000000000000000000..4dbb107edccb77dcefed6afff85d43203d133b27
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/fixtures/audioflamingo3/expected_results_batched.json
@@ -0,0 +1 @@
+{"transcriptions": ["There is no clear relationship between the barking and the music, as they seem to be independent of each other.", "(B) To indicate that language cannot express clearly, satirizing the inversion of black and white in the world"], "token_ids": [[3862, 374, 902, 2797, 5025, 1948, 279, 293, 33452, 323, 279, 4627, 11, 438, 807, 2803, 311, 387, 9489, 315, 1817, 1008, 13, 151645], [5349, 8, 2014, 13216, 429, 4128, 4157, 3158, 9355, 11, 7578, 404, 4849, 279, 46488, 315, 3691, 323, 4158, 304, 279, 1879, 151645, 151671]]}
\ No newline at end of file
diff --git a/asian_ai/performance/vLLM/tests/models/fixtures/audioflamingo3/expected_results_single.json b/asian_ai/performance/vLLM/tests/models/fixtures/audioflamingo3/expected_results_single.json
new file mode 100644
index 0000000000000000000000000000000000000000..32ba41b850b586cb5717e6bcf8e8b1ce56b78bfc
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/fixtures/audioflamingo3/expected_results_single.json
@@ -0,0 +1 @@
+{"transcriptions": ["There is no clear relationship between the barking and the music, as they seem to be independent of each other."], "token_ids": [[3862, 374, 902, 2797, 5025, 1948, 279, 293, 33452, 323, 279, 4627, 11, 438, 807, 2803, 311, 387, 9489, 315, 1817, 1008, 13, 151645]]}
diff --git a/asian_ai/performance/vLLM/tests/models/fixtures/ministral_3b_chat.json b/asian_ai/performance/vLLM/tests/models/fixtures/ministral_3b_chat.json
new file mode 100644
index 0000000000000000000000000000000000000000..22dd9527adca4c548219844e62127de360f56d88
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/fixtures/ministral_3b_chat.json
@@ -0,0 +1 @@
+[[[4380, 3937, 6122, 1261, 7244, 10575, 18970, 41132, 3923, 1408, 1261, 32656, 4691, 1454, 2246, 22131, 15179, 11521, 17277, 1046, 2], "This image shows a black dog sitting attentively on a wooden surface with its gaze directed straight ahead.", [{"4380": {"logprob": -1.0445597171783447, "rank": 1, "decoded_token": "This"}, "1784": {"logprob": -1.5445597171783447, "rank": 2, "decoded_token": "The"}, "1065": {"logprob": -1.7945597171783447, "rank": 3, "decoded_token": "A"}, "1785": {"logprob": -2.2945597171783447, "rank": 4, "decoded_token": "In"}, "4051": {"logprob": -4.357059478759766, "rank": 5, "decoded_token": "An"}}, {"3937": {"logprob": -0.012832445092499256, "rank": 1, "decoded_token": " image"}, "13083": {"logprob": -6.8253326416015625, "rank": 2, "decoded_token": " picture"}, "16649": {"logprob": -6.8878326416015625, "rank": 3, "decoded_token": " photo"}, "1395": {"logprob": -7.2003326416015625, "rank": 4, "decoded_token": " is"}, "7244": {"logprob": -7.5128326416015625, "rank": 5, "decoded_token": " black"}}, {"6122": {"logprob": -0.2629571557044983, "rank": 1, "decoded_token": " shows"}, "51948": {"logprob": -2.2629570960998535, "rank": 2, "decoded_token": " depicts"}, "6971": {"logprob": -2.5129570960998535, "rank": 3, "decoded_token": " features"}, "25981": {"logprob": -3.6379570960998535, "rank": 4, "decoded_token": " displays"}, "89995": {"logprob": -4.7004570960998535, "rank": 5, "decoded_token": " showc"}}, {"1261": {"logprob": -0.00752826826646924, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -5.132528305053711, "rank": 2, "decoded_token": " an"}, "1278": {"logprob": -6.632528305053711, "rank": 3, "decoded_token": " the"}, "7244": {"logprob": -11.195028305053711, "rank": 4, "decoded_token": " black"}, "44056": {"logprob": -11.757528305053711, "rank": 5, "decoded_token": "\ta"}}, {"7244": {"logprob": -0.19620661437511444, "rank": 1, "decoded_token": " black"}, "8500": {"logprob": -3.446206569671631, "rank": 2, "decoded_token": " dark"}, "4329": {"logprob": -3.446206569671631, "rank": 3, "decoded_token": " large"}, "6231": {"logprob": -3.446206569671631, "rank": 4, "decoded_token": " close"}, "85596": {"logprob": -4.196206569671631, "rank": 5, "decoded_token": " solemn"}}, {"10575": {"logprob": -0.08389955013990402, "rank": 1, "decoded_token": " dog"}, "119075": {"logprob": -3.14639949798584, "rank": 2, "decoded_token": " Labrador"}, "116572": {"logprob": -4.83389949798584, "rank": 3, "decoded_token": " puppy"}, "1044": {"logprob": -5.14639949798584, "rank": 4, "decoded_token": ","}, "9566": {"logprob": -5.58389949798584, "rank": 5, "decoded_token": " medium"}}, {"18970": {"logprob": -1.0850104093551636, "rank": 1, "decoded_token": " sitting"}, "1454": {"logprob": -1.6475104093551636, "rank": 2, "decoded_token": " with"}, "28528": {"logprob": -2.022510528564453, "rank": 3, "decoded_token": " lying"}, "7283": {"logprob": -2.085010528564453, "rank": 4, "decoded_token": " looking"}, "38235": {"logprob": -2.460010528564453, "rank": 5, "decoded_token": " resting"}}, {"41132": {"logprob": -1.2341952323913574, "rank": 1, "decoded_token": " attent"}, "106534": {"logprob": -1.4841952323913574, "rank": 2, "decoded_token": " calmly"}, "1505": {"logprob": -2.2966952323913574, "rank": 3, "decoded_token": " or"}, "17558": {"logprob": -2.3591952323913574, "rank": 4, "decoded_token": " closely"}, "1408": {"logprob": -2.5466952323913574, "rank": 5, "decoded_token": " on"}}, {"3923": {"logprob": -0.005889324937015772, "rank": 1, "decoded_token": "ively"}, "1556": {"logprob": -6.693389415740967, "rank": 2, "decoded_token": "ive"}, "3929": {"logprob": -6.880889415740967, "rank": 3, "decoded_token": "ently"}, "10980": {"logprob": -7.193389415740967, "rank": 4, "decoded_token": "ibly"}, "14194": {"logprob": -7.943389415740967, "rank": 5, "decoded_token": "antly"}}, {"1408": {"logprob": -0.5112090110778809, "rank": 1, "decoded_token": " on"}, "1454": {"logprob": -1.7612090110778809, "rank": 2, "decoded_token": " with"}, "1321": {"logprob": -2.386209011077881, "rank": 3, "decoded_token": " and"}, "3675": {"logprob": -2.761209011077881, "rank": 4, "decoded_token": " against"}, "1505": {"logprob": -4.198709011077881, "rank": 5, "decoded_token": " or"}}, {"1261": {"logprob": -0.189262256026268, "rank": 1, "decoded_token": " a"}, "2549": {"logprob": -2.1892621517181396, "rank": 2, "decoded_token": " what"}, "32656": {"logprob": -3.8767621517181396, "rank": 3, "decoded_token": " wooden"}, "17253": {"logprob": -4.626762390136719, "rank": 4, "decoded_token": " weather"}, "3403": {"logprob": -4.751762390136719, "rank": 5, "decoded_token": " text"}}, {"32656": {"logprob": -0.9549442529678345, "rank": 1, "decoded_token": " wooden"}, "3403": {"logprob": -1.2049442529678345, "rank": 2, "decoded_token": " text"}, "17253": {"logprob": -1.8924442529678345, "rank": 3, "decoded_token": " weather"}, "44130": {"logprob": -2.267444133758545, "rank": 4, "decoded_token": " rust"}, "16673": {"logprob": -4.142444133758545, "rank": 5, "decoded_token": " rough"}}, {"4691": {"logprob": -0.2555857002735138, "rank": 1, "decoded_token": " surface"}, "3403": {"logprob": -2.6930856704711914, "rank": 2, "decoded_token": " text"}, "1615": {"logprob": -2.8805856704711914, "rank": 3, "decoded_token": " pl"}, "1044": {"logprob": -4.193085670471191, "rank": 4, "decoded_token": ","}, "26228": {"logprob": -4.380585670471191, "rank": 5, "decoded_token": " texture"}}, {"1454": {"logprob": -0.5042062997817993, "rank": 1, "decoded_token": " with"}, "1044": {"logprob": -1.6292062997817993, "rank": 2, "decoded_token": ","}, "1046": {"logprob": -2.2542061805725098, "rank": 3, "decoded_token": "."}, "7283": {"logprob": -3.7542061805725098, "rank": 4, "decoded_token": " looking"}, "1338": {"logprob": -4.44170618057251, "rank": 5, "decoded_token": ".\n\n"}}, {"2246": {"logprob": -1.1512703895568848, "rank": 1, "decoded_token": " its"}, "1261": {"logprob": -1.2137703895568848, "rank": 2, "decoded_token": " a"}, "1420": {"logprob": -2.3387703895568848, "rank": 3, "decoded_token": " an"}, "9924": {"logprob": -2.4637703895568848, "rank": 4, "decoded_token": " wide"}, "12593": {"logprob": -3.7762703895568848, "rank": 5, "decoded_token": " slightly"}}, {"22131": {"logprob": -1.0600039958953857, "rank": 1, "decoded_token": " gaze"}, "5731": {"logprob": -1.4350039958953857, "rank": 2, "decoded_token": " eyes"}, "9924": {"logprob": -2.7475039958953857, "rank": 3, "decoded_token": " wide"}, "14781": {"logprob": -2.9350039958953857, "rank": 4, "decoded_token": " focused"}, "3518": {"logprob": -3.1225039958953857, "rank": 5, "decoded_token": " head"}}, {"15179": {"logprob": -1.0535285472869873, "rank": 1, "decoded_token": " directed"}, "9247": {"logprob": -1.5535285472869873, "rank": 2, "decoded_token": " fixed"}, "14781": {"logprob": -2.4285285472869873, "rank": 3, "decoded_token": " focused"}, "7283": {"logprob": -2.6785285472869873, "rank": 4, "decoded_token": " looking"}, "12593": {"logprob": -2.7410285472869873, "rank": 5, "decoded_token": " slightly"}}, {"11521": {"logprob": -1.4163023233413696, "rank": 1, "decoded_token": " straight"}, "40022": {"logprob": -1.6038023233413696, "rank": 2, "decoded_token": " upward"}, "74606": {"logprob": -1.6663023233413696, "rank": 3, "decoded_token": " upwards"}, "12593": {"logprob": -2.16630220413208, "rank": 4, "decoded_token": " slightly"}, "8848": {"logprob": -2.16630220413208, "rank": 5, "decoded_token": " forward"}}, {"17277": {"logprob": -0.20115239918231964, "rank": 1, "decoded_token": " ahead"}, "8848": {"logprob": -2.4511523246765137, "rank": 2, "decoded_token": " forward"}, "1513": {"logprob": -2.9511523246765137, "rank": 3, "decoded_token": " at"}, "1046": {"logprob": -4.576152324676514, "rank": 4, "decoded_token": "."}, "8994": {"logprob": -4.826152324676514, "rank": 5, "decoded_token": " towards"}}, {"1046": {"logprob": -0.1819322109222412, "rank": 1, "decoded_token": "."}, "1338": {"logprob": -2.619432210922241, "rank": 2, "decoded_token": ".\n\n"}, "1321": {"logprob": -3.119432210922241, "rank": 3, "decoded_token": " and"}, "1044": {"logprob": -4.05693244934082, "rank": 4, "decoded_token": ","}, "1626": {"logprob": -5.18193244934082, "rank": 5, "decoded_token": ".\n"}}, {"2": {"logprob": -2.0367233753204346, "rank": 1, "decoded_token": ""}, "35": {"logprob": -4.6617231369018555, "rank": 2, "decoded_token": "[/THINK]"}, "9": {"logprob": -4.9742231369018555, "rank": 3, "decoded_token": "[TOOL_CALLS]"}, "108349": {"logprob": -5.4742231369018555, "rank": 4, "decoded_token": "\u305d\u306e\u4e00\u65b9\u3067"}, "32": {"logprob": -5.8492231369018555, "rank": 5, "decoded_token": "[ARGS]"}}]], [[1784, 2158, 3937, 6122, 1261, 7244, 10575, 18970, 41132, 3923, 1408, 1261, 32656, 4691, 1454, 1261, 26517, 1321, 14781, 4818, 1338, 1784, 2667, 3937, 51948, 1261, 10726, 1290, 3719, 1307, 122203, 1044, 23745, 3591, 13194, 24361, 27469, 1294, 1278, 7786, 1454, 1295, 3506, 11223, 47260, 1408, 1278, 61263, 1046, 2], "The first image shows a black dog sitting attentively on a wooden surface with a calm and focused expression.\n\nThe second image depicts a scenic view of rugged, snow-capped mountain peaks in the distance with lush green vegetation on the slopes.", [{"1784": {"logprob": -0.6781838536262512, "rank": 1, "decoded_token": "The"}, "1049": {"logprob": -1.1781837940216064, "rank": 2, "decoded_token": "1"}, "69957": {"logprob": -4.0531840324401855, "rank": 3, "decoded_token": "Sure"}, "11745": {"logprob": -4.6781840324401855, "rank": 4, "decoded_token": "Here"}, "1785": {"logprob": -5.1781840324401855, "rank": 5, "decoded_token": "In"}}, {"2158": {"logprob": -0.17460788786411285, "rank": 1, "decoded_token": " first"}, "3937": {"logprob": -2.612107992172241, "rank": 2, "decoded_token": " image"}, "8061": {"logprob": -4.299607753753662, "rank": 3, "decoded_token": " images"}, "7244": {"logprob": -5.487107753753662, "rank": 4, "decoded_token": " black"}, "5662": {"logprob": -5.549607753753662, "rank": 5, "decoded_token": " provided"}}, {"3937": {"logprob": -0.014750940725207329, "rank": 1, "decoded_token": " image"}, "13083": {"logprob": -5.889750957489014, "rank": 2, "decoded_token": " picture"}, "2016": {"logprob": -7.264750957489014, "rank": 3, "decoded_token": " set"}, "16649": {"logprob": -7.764750957489014, "rank": 4, "decoded_token": " photo"}, "1877": {"logprob": -7.764750957489014, "rank": 5, "decoded_token": ":\n"}}, {"6122": {"logprob": -0.29441142082214355, "rank": 1, "decoded_token": " shows"}, "51948": {"logprob": -2.2944114208221436, "rank": 2, "decoded_token": " depicts"}, "1877": {"logprob": -3.4194114208221436, "rank": 3, "decoded_token": ":\n"}, "1395": {"logprob": -3.5444114208221436, "rank": 4, "decoded_token": " is"}, "25981": {"logprob": -3.7319114208221436, "rank": 5, "decoded_token": " displays"}}, {"1261": {"logprob": -0.05616755038499832, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -4.0561676025390625, "rank": 2, "decoded_token": " an"}, "1877": {"logprob": -5.8061676025390625, "rank": 3, "decoded_token": ":\n"}, "1278": {"logprob": -7.1186676025390625, "rank": 4, "decoded_token": " the"}, "20806": {"logprob": -7.6186676025390625, "rank": 5, "decoded_token": "\uff1a\n"}}, {"7244": {"logprob": -0.3175433278083801, "rank": 1, "decoded_token": " black"}, "4329": {"logprob": -2.4425432682037354, "rank": 2, "decoded_token": " large"}, "8500": {"logprob": -3.3175432682037354, "rank": 3, "decoded_token": " dark"}, "6231": {"logprob": -3.6925432682037354, "rank": 4, "decoded_token": " close"}, "85596": {"logprob": -3.7550432682037354, "rank": 5, "decoded_token": " solemn"}}, {"10575": {"logprob": -0.1241316869854927, "rank": 1, "decoded_token": " dog"}, "119075": {"logprob": -2.624131679534912, "rank": 2, "decoded_token": " Labrador"}, "116572": {"logprob": -4.686631679534912, "rank": 3, "decoded_token": " puppy"}, "15812": {"logprob": -5.436631679534912, "rank": 4, "decoded_token": " Lab"}, "1044": {"logprob": -5.561631679534912, "rank": 5, "decoded_token": ","}}, {"18970": {"logprob": -0.9899296164512634, "rank": 1, "decoded_token": " sitting"}, "7283": {"logprob": -1.8649296760559082, "rank": 2, "decoded_token": " looking"}, "28528": {"logprob": -2.114929676055908, "rank": 3, "decoded_token": " lying"}, "1454": {"logprob": -2.364929676055908, "rank": 4, "decoded_token": " with"}, "38235": {"logprob": -2.489929676055908, "rank": 5, "decoded_token": " resting"}}, {"41132": {"logprob": -1.1735738515853882, "rank": 1, "decoded_token": " attent"}, "106534": {"logprob": -1.7360738515853882, "rank": 2, "decoded_token": " calmly"}, "1408": {"logprob": -1.9235738515853882, "rank": 3, "decoded_token": " on"}, "1505": {"logprob": -2.8610739707946777, "rank": 4, "decoded_token": " or"}, "38263": {"logprob": -3.4860739707946777, "rank": 5, "decoded_token": " quietly"}}, {"3923": {"logprob": -0.011853850446641445, "rank": 1, "decoded_token": "ively"}, "1556": {"logprob": -5.6368536949157715, "rank": 2, "decoded_token": "ive"}, "3929": {"logprob": -6.6993536949157715, "rank": 3, "decoded_token": "ently"}, "10980": {"logprob": -6.6993536949157715, "rank": 4, "decoded_token": "ibly"}, "14194": {"logprob": -7.2618536949157715, "rank": 5, "decoded_token": "antly"}}, {"1408": {"logprob": -0.3410560190677643, "rank": 1, "decoded_token": " on"}, "1454": {"logprob": -2.4660561084747314, "rank": 2, "decoded_token": " with"}, "1321": {"logprob": -2.9660561084747314, "rank": 3, "decoded_token": " and"}, "3675": {"logprob": -3.2160561084747314, "rank": 4, "decoded_token": " against"}, "25644": {"logprob": -3.7785561084747314, "rank": 5, "decoded_token": " beside"}}, {"1261": {"logprob": -0.16969695687294006, "rank": 1, "decoded_token": " a"}, "2549": {"logprob": -2.7946970462799072, "rank": 2, "decoded_token": " what"}, "32656": {"logprob": -2.9821970462799072, "rank": 3, "decoded_token": " wooden"}, "3403": {"logprob": -5.107196807861328, "rank": 4, "decoded_token": " text"}, "3977": {"logprob": -5.169696807861328, "rank": 5, "decoded_token": " top"}}, {"32656": {"logprob": -0.49769073724746704, "rank": 1, "decoded_token": " wooden"}, "44130": {"logprob": -1.8726906776428223, "rank": 2, "decoded_token": " rust"}, "3403": {"logprob": -2.2476906776428223, "rank": 3, "decoded_token": " text"}, "17253": {"logprob": -2.9976906776428223, "rank": 4, "decoded_token": " weather"}, "16673": {"logprob": -3.8726906776428223, "rank": 5, "decoded_token": " rough"}}, {"4691": {"logprob": -0.42273157835006714, "rank": 1, "decoded_token": " surface"}, "1615": {"logprob": -1.985231637954712, "rank": 2, "decoded_token": " pl"}, "11237": {"logprob": -3.735231637954712, "rank": 3, "decoded_token": " floor"}, "26808": {"logprob": -3.735231637954712, "rank": 4, "decoded_token": " bench"}, "3403": {"logprob": -3.860231637954712, "rank": 5, "decoded_token": " text"}}, {"1454": {"logprob": -0.8123087286949158, "rank": 1, "decoded_token": " with"}, "1338": {"logprob": -1.5623087882995605, "rank": 2, "decoded_token": ".\n\n"}, "1626": {"logprob": -2.3748087882995605, "rank": 3, "decoded_token": ".\n"}, "7283": {"logprob": -2.8123087882995605, "rank": 4, "decoded_token": " looking"}, "1044": {"logprob": -3.5623087882995605, "rank": 5, "decoded_token": ","}}, {"1261": {"logprob": -1.0109002590179443, "rank": 1, "decoded_token": " a"}, "2246": {"logprob": -1.1984002590179443, "rank": 2, "decoded_token": " its"}, "1420": {"logprob": -2.0109002590179443, "rank": 3, "decoded_token": " an"}, "9924": {"logprob": -3.9484002590179443, "rank": 4, "decoded_token": " wide"}, "14781": {"logprob": -4.698400497436523, "rank": 5, "decoded_token": " focused"}}, {"26517": {"logprob": -2.2192542552948, "rank": 1, "decoded_token": " calm"}, "14781": {"logprob": -2.4067542552948, "rank": 2, "decoded_token": " focused"}, "11304": {"logprob": -2.6567542552948, "rank": 3, "decoded_token": " serious"}, "29691": {"logprob": -2.7192542552948, "rank": 4, "decoded_token": " contempl"}, "12593": {"logprob": -3.1567542552948, "rank": 5, "decoded_token": " slightly"}}, {"1321": {"logprob": -0.8886916637420654, "rank": 1, "decoded_token": " and"}, "4818": {"logprob": -1.0136916637420654, "rank": 2, "decoded_token": " expression"}, "1044": {"logprob": -2.2636916637420654, "rank": 3, "decoded_token": ","}, "22131": {"logprob": -3.1386916637420654, "rank": 4, "decoded_token": " gaze"}, "1311": {"logprob": -3.9511916637420654, "rank": 5, "decoded_token": " de"}}, {"14781": {"logprob": -1.208945870399475, "rank": 1, "decoded_token": " focused"}, "38462": {"logprob": -1.583945870399475, "rank": 2, "decoded_token": " curious"}, "97680": {"logprob": -2.9589457511901855, "rank": 3, "decoded_token": " thoughtful"}, "11304": {"logprob": -2.9589457511901855, "rank": 4, "decoded_token": " serious"}, "29691": {"logprob": -3.3964457511901855, "rank": 5, "decoded_token": " contempl"}}, {"4818": {"logprob": -0.16998574137687683, "rank": 1, "decoded_token": " expression"}, "22131": {"logprob": -2.669985771179199, "rank": 2, "decoded_token": " gaze"}, "2985": {"logprob": -3.232485771179199, "rank": 3, "decoded_token": " look"}, "1311": {"logprob": -3.919985771179199, "rank": 4, "decoded_token": " de"}, "13988": {"logprob": -4.982485771179199, "rank": 5, "decoded_token": " appearance"}}, {"1338": {"logprob": -0.4974508285522461, "rank": 1, "decoded_token": ".\n\n"}, "1626": {"logprob": -1.372450828552246, "rank": 2, "decoded_token": ".\n"}, "1046": {"logprob": -3.497450828552246, "rank": 3, "decoded_token": "."}, "1044": {"logprob": -4.122450828552246, "rank": 4, "decoded_token": ","}, "12560": {"logprob": -4.934950828552246, "rank": 5, "decoded_token": "\u0589\n\n"}}, {"1784": {"logprob": -0.05534925311803818, "rank": 1, "decoded_token": "The"}, "1785": {"logprob": -4.305349349975586, "rank": 2, "decoded_token": "In"}, "6958": {"logprob": -5.617849349975586, "rank": 3, "decoded_token": "There"}, "84593": {"logprob": -5.617849349975586, "rank": 4, "decoded_token": "_The"}, "4393": {"logprob": -6.180349349975586, "rank": 5, "decoded_token": "For"}}, {"2667": {"logprob": -0.035550788044929504, "rank": 1, "decoded_token": " second"}, "3937": {"logprob": -6.160550594329834, "rank": 2, "decoded_token": " image"}, "13023": {"logprob": -6.348050594329834, "rank": 3, "decoded_token": "second"}, "6360": {"logprob": -7.660550594329834, "rank": 4, "decoded_token": " description"}, "2158": {"logprob": -7.785550594329834, "rank": 5, "decoded_token": " first"}}, {"3937": {"logprob": -0.045355767011642456, "rank": 1, "decoded_token": " image"}, "13083": {"logprob": -5.857855796813965, "rank": 2, "decoded_token": " picture"}, "16649": {"logprob": -6.295355796813965, "rank": 3, "decoded_token": " photo"}, "2016": {"logprob": -6.482855796813965, "rank": 4, "decoded_token": " set"}, "5662": {"logprob": -6.857855796813965, "rank": 5, "decoded_token": " provided"}}, {"51948": {"logprob": -0.5656330585479736, "rank": 1, "decoded_token": " depicts"}, "25981": {"logprob": -1.8156330585479736, "rank": 2, "decoded_token": " displays"}, "66583": {"logprob": -2.5031330585479736, "rank": 3, "decoded_token": " captures"}, "6971": {"logprob": -2.7531330585479736, "rank": 4, "decoded_token": " features"}, "1395": {"logprob": -3.3781330585479736, "rank": 5, "decoded_token": " is"}}, {"1261": {"logprob": -0.13412977755069733, "rank": 1, "decoded_token": " a"}, "122203": {"logprob": -2.946629762649536, "rank": 2, "decoded_token": " rugged"}, "1420": {"logprob": -3.821629762649536, "rank": 3, "decoded_token": " an"}, "10726": {"logprob": -4.946630001068115, "rank": 4, "decoded_token": " scen"}, "13770": {"logprob": -5.196630001068115, "rank": 5, "decoded_token": " maj"}}, {"10726": {"logprob": -0.7839541435241699, "rank": 1, "decoded_token": " scen"}, "37849": {"logprob": -2.47145414352417, "rank": 2, "decoded_token": " breat"}, "122203": {"logprob": -2.72145414352417, "rank": 3, "decoded_token": " rugged"}, "23874": {"logprob": -2.72145414352417, "rank": 4, "decoded_token": " pictures"}, "15375": {"logprob": -3.47145414352417, "rank": 5, "decoded_token": " vast"}}, {"1290": {"logprob": -0.000259365770034492, "rank": 1, "decoded_token": "ic"}, "2981": {"logprob": -9.187759399414062, "rank": 2, "decoded_token": "ically"}, "1702": {"logprob": -11.250259399414062, "rank": 3, "decoded_token": "ice"}, "16832": {"logprob": -12.375259399414062, "rank": 4, "decoded_token": "...\n"}, "1685": {"logprob": -12.500259399414062, "rank": 5, "decoded_token": "ical"}}, {"3719": {"logprob": -0.9477202892303467, "rank": 1, "decoded_token": " view"}, "24361": {"logprob": -1.3227202892303467, "rank": 2, "decoded_token": " mountain"}, "127945": {"logprob": -1.9477202892303467, "rank": 3, "decoded_token": " mountainous"}, "1044": {"logprob": -3.0102202892303467, "rank": 4, "decoded_token": ","}, "28035": {"logprob": -3.0727202892303467, "rank": 5, "decoded_token": " landscape"}}, {"1307": {"logprob": -0.030216755345463753, "rank": 1, "decoded_token": " of"}, "1562": {"logprob": -4.217716693878174, "rank": 2, "decoded_token": " from"}, "24018": {"logprob": -5.717716693878174, "rank": 3, "decoded_token": " featuring"}, "11050": {"logprob": -6.217716693878174, "rank": 4, "decoded_token": " showing"}, "2015": {"logprob": -6.280216693878174, "rank": 5, "decoded_token": " up"}}, {"122203": {"logprob": -1.1189241409301758, "rank": 1, "decoded_token": " rugged"}, "1261": {"logprob": -1.6189241409301758, "rank": 2, "decoded_token": " a"}, "127945": {"logprob": -2.681424140930176, "rank": 3, "decoded_token": " mountainous"}, "11223": {"logprob": -2.931424140930176, "rank": 4, "decoded_token": " green"}, "23745": {"logprob": -3.056424140930176, "rank": 5, "decoded_token": " snow"}}, {"1044": {"logprob": -1.033378005027771, "rank": 1, "decoded_token": ","}, "24361": {"logprob": -1.158378005027771, "rank": 2, "decoded_token": " mountain"}, "35463": {"logprob": -1.658378005027771, "rank": 3, "decoded_token": " mountains"}, "127945": {"logprob": -2.2833781242370605, "rank": 4, "decoded_token": " mountainous"}, "1321": {"logprob": -4.9708781242370605, "rank": 5, "decoded_token": " and"}}, {"23745": {"logprob": -0.8830229043960571, "rank": 1, "decoded_token": " snow"}, "127945": {"logprob": -1.8830229043960571, "rank": 2, "decoded_token": " mountainous"}, "11223": {"logprob": -1.8830229043960571, "rank": 3, "decoded_token": " green"}, "1394": {"logprob": -2.5705227851867676, "rank": 4, "decoded_token": " for"}, "95746": {"logprob": -3.5080227851867676, "rank": 5, "decoded_token": " rocky"}}, {"3591": {"logprob": -0.5006332397460938, "rank": 1, "decoded_token": "-c"}, "114525": {"logprob": -1.1256332397460938, "rank": 2, "decoded_token": "-covered"}, "18928": {"logprob": -3.5006332397460938, "rank": 3, "decoded_token": "-top"}, "1099": {"logprob": -4.500633239746094, "rank": 4, "decoded_token": "c"}, "24263": {"logprob": -4.938133239746094, "rank": 5, "decoded_token": "-cl"}}, {"13194": {"logprob": -0.007475971709936857, "rank": 1, "decoded_token": "apped"}, "36649": {"logprob": -6.132475852966309, "rank": 2, "decoded_token": "rowned"}, "13234": {"logprob": -6.694975852966309, "rank": 3, "decoded_token": "aped"}, "10261": {"logprob": -6.819975852966309, "rank": 4, "decoded_token": "rest"}, "4681": {"logprob": -7.757475852966309, "rank": 5, "decoded_token": "ored"}}, {"24361": {"logprob": -0.6446366906166077, "rank": 1, "decoded_token": " mountain"}, "35463": {"logprob": -0.8946366906166077, "rank": 2, "decoded_token": " mountains"}, "127945": {"logprob": -3.394636631011963, "rank": 3, "decoded_token": " mountainous"}, "116555": {"logprob": -4.894636631011963, "rank": 4, "decoded_token": " alpine"}, "1321": {"logprob": -4.894636631011963, "rank": 5, "decoded_token": " and"}}, {"27469": {"logprob": -0.12543931603431702, "rank": 1, "decoded_token": " peaks"}, "26236": {"logprob": -2.625439405441284, "rank": 2, "decoded_token": " ranges"}, "103398": {"logprob": -3.937939405441284, "rank": 3, "decoded_token": " ridges"}, "24765": {"logprob": -4.875439167022705, "rank": 4, "decoded_token": " terrain"}, "84497": {"logprob": -5.187939167022705, "rank": 5, "decoded_token": " landscapes"}}, {"1294": {"logprob": -1.4923679828643799, "rank": 1, "decoded_token": " in"}, "1454": {"logprob": -1.8673679828643799, "rank": 2, "decoded_token": " with"}, "29817": {"logprob": -2.55486798286438, "rank": 3, "decoded_token": " surrounded"}, "26619": {"logprob": -2.61736798286438, "rank": 4, "decoded_token": " rising"}, "1321": {"logprob": -2.61736798286438, "rank": 5, "decoded_token": " and"}}, {"1278": {"logprob": -0.3165818154811859, "rank": 1, "decoded_token": " the"}, "1261": {"logprob": -1.4415818452835083, "rank": 2, "decoded_token": " a"}, "1420": {"logprob": -4.316581726074219, "rank": 3, "decoded_token": " an"}, "5561": {"logprob": -5.754081726074219, "rank": 4, "decoded_token": " various"}, "2549": {"logprob": -6.066581726074219, "rank": 5, "decoded_token": " what"}}, {"7786": {"logprob": -0.5495827198028564, "rank": 1, "decoded_token": " distance"}, "7042": {"logprob": -1.0495827198028564, "rank": 2, "decoded_token": " background"}, "30594": {"logprob": -3.1745827198028564, "rank": 3, "decoded_token": " distant"}, "115381": {"logprob": -5.237082481384277, "rank": 4, "decoded_token": " Alps"}, "92504": {"logprob": -5.237082481384277, "rank": 5, "decoded_token": " backdrop"}}, {"1454": {"logprob": -0.6429675817489624, "rank": 1, "decoded_token": " with"}, "1044": {"logprob": -2.205467700958252, "rank": 2, "decoded_token": ","}, "3675": {"logprob": -2.330467700958252, "rank": 3, "decoded_token": " against"}, "1046": {"logprob": -3.330467700958252, "rank": 4, "decoded_token": "."}, "2136": {"logprob": -3.392967700958252, "rank": 5, "decoded_token": " over"}}, {"1295": {"logprob": -1.4274311065673828, "rank": 1, "decoded_token": " l"}, "1261": {"logprob": -1.6774311065673828, "rank": 2, "decoded_token": " a"}, "11223": {"logprob": -1.9274311065673828, "rank": 3, "decoded_token": " green"}, "47147": {"logprob": -3.239931106567383, "rank": 4, "decoded_token": " steep"}, "50373": {"logprob": -3.302431106567383, "rank": 5, "decoded_token": " patches"}}, {"3506": {"logprob": -0.0006933192489668727, "rank": 1, "decoded_token": "ush"}, "16938": {"logprob": -8.563193321228027, "rank": 2, "decoded_token": "usher"}, "1374": {"logprob": -9.188193321228027, "rank": 3, "decoded_token": "us"}, "90716": {"logprob": -9.563193321228027, "rank": 4, "decoded_token": "USH"}, "5245": {"logprob": -9.688193321228027, "rank": 5, "decoded_token": "acy"}}, {"11223": {"logprob": -0.5139672160148621, "rank": 1, "decoded_token": " green"}, "1044": {"logprob": -1.5764672756195068, "rank": 2, "decoded_token": ","}, "4174": {"logprob": -2.451467275619507, "rank": 3, "decoded_token": " gre"}, "47260": {"logprob": -3.451467275619507, "rank": 4, "decoded_token": " vegetation"}, "1394": {"logprob": -4.763967037200928, "rank": 5, "decoded_token": " for"}}, {"47260": {"logprob": -1.0371246337890625, "rank": 1, "decoded_token": " vegetation"}, "61263": {"logprob": -1.8496246337890625, "rank": 2, "decoded_token": " slopes"}, "94549": {"logprob": -1.9121246337890625, "rank": 3, "decoded_token": " valleys"}, "50373": {"logprob": -3.4121246337890625, "rank": 4, "decoded_token": " patches"}, "4953": {"logprob": -3.5371246337890625, "rank": 5, "decoded_token": " lower"}}, {"1408": {"logprob": -1.3317866325378418, "rank": 1, "decoded_token": " on"}, "1294": {"logprob": -1.7067866325378418, "rank": 2, "decoded_token": " in"}, "1321": {"logprob": -2.394286632537842, "rank": 3, "decoded_token": " and"}, "5956": {"logprob": -2.644286632537842, "rank": 4, "decoded_token": " below"}, "1513": {"logprob": -2.706786632537842, "rank": 5, "decoded_token": " at"}}, {"1278": {"logprob": -0.6638099551200867, "rank": 1, "decoded_token": " the"}, "1261": {"logprob": -2.1638100147247314, "rank": 2, "decoded_token": " a"}, "47147": {"logprob": -2.6013100147247314, "rank": 3, "decoded_token": " steep"}, "4953": {"logprob": -2.9138100147247314, "rank": 4, "decoded_token": " lower"}, "95746": {"logprob": -3.0388100147247314, "rank": 5, "decoded_token": " rocky"}}, {"61263": {"logprob": -1.3242369890213013, "rank": 1, "decoded_token": " slopes"}, "95746": {"logprob": -2.3242368698120117, "rank": 2, "decoded_token": " rocky"}, "4953": {"logprob": -2.3867368698120117, "rank": 3, "decoded_token": " lower"}, "79831": {"logprob": -2.7617368698120117, "rank": 4, "decoded_token": " foreground"}, "47147": {"logprob": -2.8867368698120117, "rank": 5, "decoded_token": " steep"}}, {"1046": {"logprob": -0.7772958278656006, "rank": 1, "decoded_token": "."}, "5956": {"logprob": -1.1522958278656006, "rank": 2, "decoded_token": " below"}, "1294": {"logprob": -2.3397958278656006, "rank": 3, "decoded_token": " in"}, "1321": {"logprob": -3.5897958278656006, "rank": 4, "decoded_token": " and"}, "1307": {"logprob": -4.27729606628418, "rank": 5, "decoded_token": " of"}}, {"2": {"logprob": -0.031993232667446136, "rank": 1, "decoded_token": ""}, "3730": {"logprob": -7.281993389129639, "rank": 2, "decoded_token": " There"}, "1531": {"logprob": -7.281993389129639, "rank": 3, "decoded_token": " The"}, "2409": {"logprob": -8.65699291229248, "rank": 4, "decoded_token": " This"}, "2157": {"logprob": -8.65699291229248, "rank": 5, "decoded_token": " It"}}]], [[1049, 1046, 1531, 2158, 3937, 6122, 1261, 7244, 10575, 18970, 41132, 3923, 1408, 1261, 32656, 4691, 1626, 1256, 1462, 1531, 2667, 3937, 1319, 3715, 4326, 1294, 2143, 4098, 1041, 10249, 1317, 1402, 1261, 24361, 28035, 1454, 122203, 24765, 1321, 30594, 27469, 1338, 1050, 1046, 1531, 5888, 3937, 51948, 1261, 2169, 2509, 29397, 13327, 1454, 26905, 22140, 11981, 1278, 46422, 1321, 1261, 92731, 2965, 19710, 4837, 1278, 46422, 1338, 1051, 1046, 1531, 12432, 3937, 6971, 1261, 53301, 59396, 3549, 121040, 1536, 23170, 1321, 16429, 1294, 1261, 23874, 1872, 41730, 9436, 1338, 1052, 1046, 1531, 19723, 3937, 1319, 3715, 24512, 1435, 1651, 1278, 5719, 4546, 1041, 2168, 1402, 1278, 1925, 1454, 1278, 10575, 2790, 1044, 1809, 4136, 1494, 1681, 5314, 5055, 1044, 3226, 2190, 1261, 2801, 6468, 1693, 1729, 3369, 1278, 3629, 75275, 1877, 1256, 1462, 1531, 5719, 5662, 3937, 1395, 1261, 7244, 10575, 7283, 2015, 1454, 1261, 26517, 4818, 1408, 1261, 44130, 1290, 32656, 1615, 2395, 7042, 1338, 2892, 38695, 1044, 3226, 2190, 1278, 6298, 6360, 1394, 1278, 5662, 7244, 10575, 3937, 1877, 1065, 7244, 10575, 11589, 1264, 1935, 3929, 40022, 1454, 1261, 26517, 4818, 1408, 1261, 44130, 1290, 32656, 1615, 2395, 7042, 1046, 2], "1. The first image shows a black dog sitting attentively on a wooden surface.\n - The second image (not shown in your question) appears to be a mountain landscape with rugged terrain and distant peaks.\n\n2. The third image depicts a serene beach scene with gentle waves meeting the shore and a lone person walking along the shore.\n\n3. The fourth image features a winding gravel path bordered by grass and trees in a picturesque outdoor setting.\n\n4. The fifth image (not applicable as per the initial request) would be the one with the dog again, but since it's already described, here\u2019s a different focus if we consider the following hypothetical:\n - The initial provided image is a black dog looking up with a calm expression on a rustic wooden plank background.\n\nTo clarify, here\u2019s the correct description for the provided black dog image:\nA black dog gazes intently upward with a calm expression on a rustic wooden plank background.", [{"1049": {"logprob": -0.3098178803920746, "rank": 1, "decoded_token": "1"}, "1784": {"logprob": -2.1848177909851074, "rank": 2, "decoded_token": "The"}, "11745": {"logprob": -2.6223177909851074, "rank": 3, "decoded_token": "Here"}, "2757": {"logprob": -6.059817790985107, "rank": 4, "decoded_token": "It"}, "69957": {"logprob": -6.122317790985107, "rank": 5, "decoded_token": "Sure"}}, {"1046": {"logprob": -0.16265615820884705, "rank": 1, "decoded_token": "."}, "1626": {"logprob": -6.475156307220459, "rank": 2, "decoded_token": ".\n"}, "1314": {"logprob": -6.975156307220459, "rank": 3, "decoded_token": "st"}, "1319": {"logprob": -7.412656307220459, "rank": 4, "decoded_token": " ("}, "27": {"logprob": -7.725156307220459, "rank": 5, "decoded_token": ""}}, {"1531": {"logprob": -0.5334714651107788, "rank": 1, "decoded_token": " The"}, "11967": {"logprob": -3.9709715843200684, "rank": 2, "decoded_token": " Image"}, "1349": {"logprob": -3.9709715843200684, "rank": 3, "decoded_token": " A"}, "7610": {"logprob": -3.9709715843200684, "rank": 4, "decoded_token": " First"}, "2048": {"logprob": -4.345971584320068, "rank": 5, "decoded_token": " An"}}, {"2158": {"logprob": -0.4163813889026642, "rank": 1, "decoded_token": " first"}, "3937": {"logprob": -1.2913813591003418, "rank": 2, "decoded_token": " image"}, "7244": {"logprob": -3.853881359100342, "rank": 3, "decoded_token": " black"}, "13083": {"logprob": -4.166381359100342, "rank": 4, "decoded_token": " picture"}, "16649": {"logprob": -5.103881359100342, "rank": 5, "decoded_token": " photo"}}, {"3937": {"logprob": -0.09788376092910767, "rank": 1, "decoded_token": " image"}, "13083": {"logprob": -3.785383701324463, "rank": 2, "decoded_token": " picture"}, "2016": {"logprob": -4.410383701324463, "rank": 3, "decoded_token": " set"}, "1319": {"logprob": -4.472883701324463, "rank": 4, "decoded_token": " ("}, "5662": {"logprob": -4.847883701324463, "rank": 5, "decoded_token": " provided"}}, {"6122": {"logprob": -0.3766213655471802, "rank": 1, "decoded_token": " shows"}, "1395": {"logprob": -2.1266212463378906, "rank": 2, "decoded_token": " is"}, "51948": {"logprob": -2.6891212463378906, "rank": 3, "decoded_token": " depicts"}, "6971": {"logprob": -3.4391212463378906, "rank": 4, "decoded_token": " features"}, "1058": {"logprob": -3.5641212463378906, "rank": 5, "decoded_token": ":"}}, {"1261": {"logprob": -0.04542229697108269, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -3.170422315597534, "rank": 2, "decoded_token": " an"}, "1278": {"logprob": -6.732922077178955, "rank": 3, "decoded_token": " the"}, "1877": {"logprob": -8.795422554016113, "rank": 4, "decoded_token": ":\n"}, "7244": {"logprob": -9.232922554016113, "rank": 5, "decoded_token": " black"}}, {"7244": {"logprob": -0.45382726192474365, "rank": 1, "decoded_token": " black"}, "4329": {"logprob": -2.266327381134033, "rank": 2, "decoded_token": " large"}, "85596": {"logprob": -3.141327381134033, "rank": 3, "decoded_token": " solemn"}, "8500": {"logprob": -3.203827381134033, "rank": 4, "decoded_token": " dark"}, "16450": {"logprob": -3.766327381134033, "rank": 5, "decoded_token": " sle"}}, {"10575": {"logprob": -0.09157121181488037, "rank": 1, "decoded_token": " dog"}, "119075": {"logprob": -3.27907133102417, "rank": 2, "decoded_token": " Labrador"}, "116572": {"logprob": -4.40407133102417, "rank": 3, "decoded_token": " puppy"}, "15812": {"logprob": -5.40407133102417, "rank": 4, "decoded_token": " Lab"}, "1044": {"logprob": -5.52907133102417, "rank": 5, "decoded_token": ","}}, {"18970": {"logprob": -0.9958467483520508, "rank": 1, "decoded_token": " sitting"}, "7283": {"logprob": -2.058346748352051, "rank": 2, "decoded_token": " looking"}, "28528": {"logprob": -2.120846748352051, "rank": 3, "decoded_token": " lying"}, "38235": {"logprob": -2.183346748352051, "rank": 4, "decoded_token": " resting"}, "1454": {"logprob": -2.558346748352051, "rank": 5, "decoded_token": " with"}}, {"41132": {"logprob": -1.2059022188186646, "rank": 1, "decoded_token": " attent"}, "1408": {"logprob": -1.8309022188186646, "rank": 2, "decoded_token": " on"}, "106534": {"logprob": -1.8934022188186646, "rank": 3, "decoded_token": " calmly"}, "1321": {"logprob": -3.268402099609375, "rank": 4, "decoded_token": " and"}, "1454": {"logprob": -3.330902099609375, "rank": 5, "decoded_token": " with"}}, {"3923": {"logprob": -0.005489394534379244, "rank": 1, "decoded_token": "ively"}, "1556": {"logprob": -5.505489349365234, "rank": 2, "decoded_token": "ive"}, "3929": {"logprob": -7.505489349365234, "rank": 3, "decoded_token": "ently"}, "10980": {"logprob": -8.505489349365234, "rank": 4, "decoded_token": "ibly"}, "14194": {"logprob": -9.130489349365234, "rank": 5, "decoded_token": "antly"}}, {"1408": {"logprob": -0.3085399270057678, "rank": 1, "decoded_token": " on"}, "1454": {"logprob": -2.433539867401123, "rank": 2, "decoded_token": " with"}, "1321": {"logprob": -3.308539867401123, "rank": 3, "decoded_token": " and"}, "25644": {"logprob": -3.496039867401123, "rank": 4, "decoded_token": " beside"}, "3675": {"logprob": -3.746039867401123, "rank": 5, "decoded_token": " against"}}, {"1261": {"logprob": -0.17512622475624084, "rank": 1, "decoded_token": " a"}, "32656": {"logprob": -2.550126314163208, "rank": 2, "decoded_token": " wooden"}, "2549": {"logprob": -3.800126314163208, "rank": 3, "decoded_token": " what"}, "44130": {"logprob": -4.675126075744629, "rank": 4, "decoded_token": " rust"}, "17253": {"logprob": -4.862626075744629, "rank": 5, "decoded_token": " weather"}}, {"32656": {"logprob": -0.5213224291801453, "rank": 1, "decoded_token": " wooden"}, "44130": {"logprob": -1.70882248878479, "rank": 2, "decoded_token": " rust"}, "3403": {"logprob": -2.45882248878479, "rank": 3, "decoded_token": " text"}, "17253": {"logprob": -2.89632248878479, "rank": 4, "decoded_token": " weather"}, "16673": {"logprob": -3.89632248878479, "rank": 5, "decoded_token": " rough"}}, {"4691": {"logprob": -0.7510372996330261, "rank": 1, "decoded_token": " surface"}, "1615": {"logprob": -1.188537359237671, "rank": 2, "decoded_token": " pl"}, "9710": {"logprob": -3.751037359237671, "rank": 3, "decoded_token": " board"}, "3403": {"logprob": -3.751037359237671, "rank": 4, "decoded_token": " text"}, "11237": {"logprob": -3.938537359237671, "rank": 5, "decoded_token": " floor"}}, {"1626": {"logprob": -0.5343737006187439, "rank": 1, "decoded_token": ".\n"}, "1454": {"logprob": -1.4093737602233887, "rank": 2, "decoded_token": " with"}, "1338": {"logprob": -2.7843737602233887, "rank": 3, "decoded_token": ".\n\n"}, "7283": {"logprob": -3.5343737602233887, "rank": 4, "decoded_token": " looking"}, "1044": {"logprob": -3.8468737602233887, "rank": 5, "decoded_token": ","}}, {"1256": {"logprob": -1.0727235078811646, "rank": 1, "decoded_token": " "}, "1050": {"logprob": -1.4477235078811646, "rank": 2, "decoded_token": "2"}, "1293": {"logprob": -2.947723388671875, "rank": 3, "decoded_token": " "}, "6837": {"logprob": -5.260223388671875, "rank": 4, "decoded_token": "\u06f2"}, "1260": {"logprob": -5.822723388671875, "rank": 5, "decoded_token": " "}}, {"1462": {"logprob": -2.970078468322754, "rank": 1, "decoded_token": " -"}, "1319": {"logprob": -3.470078468322754, "rank": 2, "decoded_token": " ("}, "1032": {"logprob": -4.220078468322754, "rank": 3, "decoded_token": " "}, "49958": {"logprob": -4.720078468322754, "rank": 4, "decoded_token": " ```\n"}, "9380": {"logprob": -5.282578468322754, "rank": 5, "decoded_token": " Here"}}, {"1531": {"logprob": -1.9009761810302734, "rank": 1, "decoded_token": " The"}, "10322": {"logprob": -2.9634761810302734, "rank": 2, "decoded_token": " Second"}, "1319": {"logprob": -3.1509761810302734, "rank": 3, "decoded_token": " ("}, "9380": {"logprob": -3.3384761810302734, "rank": 4, "decoded_token": " Here"}, "11967": {"logprob": -3.4634761810302734, "rank": 5, "decoded_token": " Image"}}, {"2667": {"logprob": -0.49421006441116333, "rank": 1, "decoded_token": " second"}, "3937": {"logprob": -2.9942100048065186, "rank": 2, "decoded_token": " image"}, "7244": {"logprob": -3.6192100048065186, "rank": 3, "decoded_token": " black"}, "13827": {"logprob": -3.7442100048065186, "rank": 4, "decoded_token": " subsequent"}, "15115": {"logprob": -3.8692100048065186, "rank": 5, "decoded_token": " detailed"}}, {"3937": {"logprob": -0.938382625579834, "rank": 1, "decoded_token": " image"}, "2016": {"logprob": -2.500882625579834, "rank": 2, "decoded_token": " set"}, "1319": {"logprob": -2.938382625579834, "rank": 3, "decoded_token": " ("}, "5662": {"logprob": -3.000882625579834, "rank": 4, "decoded_token": " provided"}, "7293": {"logprob": -3.438382625579834, "rank": 5, "decoded_token": " actual"}}, {"1319": {"logprob": -2.159437417984009, "rank": 3, "decoded_token": " ("}, "51948": {"logprob": -2.159437417984009, "rank": 1, "decoded_token": " depicts"}, "1395": {"logprob": -2.159437417984009, "rank": 2, "decoded_token": " is"}, "10249": {"logprob": -2.659437417984009, "rank": 4, "decoded_token": " appears"}, "1058": {"logprob": -2.784437417984009, "rank": 5, "decoded_token": ":"}}, {"3715": {"logprob": -2.364915370941162, "rank": 1, "decoded_token": "not"}, "11018": {"logprob": -2.614915370941162, "rank": 2, "decoded_token": "which"}, "2914": {"logprob": -2.927415370941162, "rank": 3, "decoded_token": "inc"}, "39575": {"logprob": -3.239915370941162, "rank": 4, "decoded_token": "mist"}, "3265": {"logprob": -3.302415370941162, "rank": 5, "decoded_token": "the"}}, {"4326": {"logprob": -1.9997965097427368, "rank": 1, "decoded_token": " shown"}, "24512": {"logprob": -2.2497963905334473, "rank": 2, "decoded_token": " applicable"}, "5662": {"logprob": -2.8747963905334473, "rank": 3, "decoded_token": " provided"}, "13874": {"logprob": -2.9372963905334473, "rank": 4, "decoded_token": " visible"}, "1278": {"logprob": -3.1247963905334473, "rank": 5, "decoded_token": " the"}}, {"1294": {"logprob": -1.0730445384979248, "rank": 1, "decoded_token": " in"}, "1041": {"logprob": -2.260544538497925, "rank": 2, "decoded_token": ")"}, "4244": {"logprob": -2.448044538497925, "rank": 3, "decoded_token": "):"}, "3226": {"logprob": -2.635544538497925, "rank": 4, "decoded_token": " here"}, "1435": {"logprob": -3.135544538497925, "rank": 5, "decoded_token": " as"}}, {"2143": {"logprob": -0.9278546571731567, "rank": 1, "decoded_token": " your"}, "1278": {"logprob": -1.2403546571731567, "rank": 2, "decoded_token": " the"}, "4098": {"logprob": -2.177854537963867, "rank": 3, "decoded_token": " question"}, "12705": {"logprob": -3.740354537963867, "rank": 4, "decoded_token": " detail"}, "1593": {"logprob": -3.927854537963867, "rank": 5, "decoded_token": " this"}}, {"4098": {"logprob": -1.8063793182373047, "rank": 1, "decoded_token": " question"}, "4546": {"logprob": -2.3063793182373047, "rank": 2, "decoded_token": " request"}, "4618": {"logprob": -2.4313793182373047, "rank": 3, "decoded_token": " original"}, "7330": {"logprob": -2.5563793182373047, "rank": 4, "decoded_token": " query"}, "5719": {"logprob": -2.8688793182373047, "rank": 5, "decoded_token": " initial"}}, {"1041": {"logprob": -1.411703109741211, "rank": 1, "decoded_token": ")"}, "4244": {"logprob": -2.099203109741211, "rank": 2, "decoded_token": "):"}, "1809": {"logprob": -2.536703109741211, "rank": 3, "decoded_token": " but"}, "1044": {"logprob": -2.661703109741211, "rank": 4, "decoded_token": ","}, "1681": {"logprob": -3.474203109741211, "rank": 5, "decoded_token": "'s"}}, {"10249": {"logprob": -1.607055425643921, "rank": 1, "decoded_token": " appears"}, "51948": {"logprob": -1.982055425643921, "rank": 2, "decoded_token": " depicts"}, "7444": {"logprob": -2.044555425643921, "rank": 3, "decoded_token": " seems"}, "1395": {"logprob": -2.169555425643921, "rank": 4, "decoded_token": " is"}, "2168": {"logprob": -2.982055425643921, "rank": 5, "decoded_token": " would"}}, {"1317": {"logprob": -0.1792934685945511, "rank": 1, "decoded_token": " to"}, "51723": {"logprob": -4.116793632507324, "rank": 2, "decoded_token": " unrelated"}, "5711": {"logprob": -4.616793632507324, "rank": 3, "decoded_token": " mis"}, "73751": {"logprob": -4.804293632507324, "rank": 4, "decoded_token": " incorrectly"}, "1605": {"logprob": -4.866793632507324, "rank": 5, "decoded_token": " not"}}, {"1402": {"logprob": -0.8325714468955994, "rank": 1, "decoded_token": " be"}, "17767": {"logprob": -1.4575715065002441, "rank": 2, "decoded_token": " depict"}, "7326": {"logprob": -3.395071506500244, "rank": 3, "decoded_token": " feature"}, "1736": {"logprob": -3.520071506500244, "rank": 4, "decoded_token": " have"}, "32688": {"logprob": -3.582571506500244, "rank": 5, "decoded_token": " illustrate"}}, {"1261": {"logprob": -1.2619296312332153, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -2.824429512023926, "rank": 2, "decoded_token": " an"}, "51723": {"logprob": -3.261929512023926, "rank": 3, "decoded_token": " unrelated"}, "25342": {"logprob": -3.511929512023926, "rank": 4, "decoded_token": " incorrect"}, "1307": {"logprob": -3.511929512023926, "rank": 5, "decoded_token": " of"}}, {"24361": {"logprob": -2.2388875484466553, "rank": 1, "decoded_token": " mountain"}, "10726": {"logprob": -2.5513875484466553, "rank": 2, "decoded_token": " scen"}, "127945": {"logprob": -3.3013875484466553, "rank": 3, "decoded_token": " mountainous"}, "15115": {"logprob": -3.6138875484466553, "rank": 4, "decoded_token": " detailed"}, "3719": {"logprob": -3.7388875484466553, "rank": 5, "decoded_token": " view"}}, {"28035": {"logprob": -0.8160803318023682, "rank": 1, "decoded_token": " landscape"}, "4521": {"logprob": -1.2535803318023682, "rank": 2, "decoded_token": " range"}, "3719": {"logprob": -2.691080331802368, "rank": 3, "decoded_token": " view"}, "13327": {"logprob": -3.253580331802368, "rank": 4, "decoded_token": " scene"}, "1454": {"logprob": -3.566080331802368, "rank": 5, "decoded_token": " with"}}, {"1454": {"logprob": -0.6144524812698364, "rank": 1, "decoded_token": " with"}, "1626": {"logprob": -2.051952362060547, "rank": 2, "decoded_token": ".\n"}, "1338": {"logprob": -2.176952362060547, "rank": 3, "decoded_token": ".\n\n"}, "3719": {"logprob": -3.926952362060547, "rank": 4, "decoded_token": " view"}, "1562": {"logprob": -3.989452362060547, "rank": 5, "decoded_token": " from"}}, {"122203": {"logprob": -1.7088322639465332, "rank": 1, "decoded_token": " rugged"}, "23745": {"logprob": -2.208832263946533, "rank": 2, "decoded_token": " snow"}, "11223": {"logprob": -2.521332263946533, "rank": 3, "decoded_token": " green"}, "27469": {"logprob": -2.833832263946533, "rank": 4, "decoded_token": " peaks"}, "47147": {"logprob": -2.896332263946533, "rank": 5, "decoded_token": " steep"}}, {"24765": {"logprob": -0.6277848482131958, "rank": 1, "decoded_token": " terrain"}, "27469": {"logprob": -1.2527848482131958, "rank": 2, "decoded_token": " peaks"}, "1044": {"logprob": -2.9402847290039062, "rank": 3, "decoded_token": ","}, "130655": {"logprob": -3.6277847290039062, "rank": 4, "decoded_token": " cliffs"}, "57912": {"logprob": -4.002784729003906, "rank": 5, "decoded_token": " terrains"}}, {"1321": {"logprob": -0.6374254822731018, "rank": 1, "decoded_token": " and"}, "1626": {"logprob": -1.762425422668457, "rank": 2, "decoded_token": ".\n"}, "1338": {"logprob": -1.762425422668457, "rank": 3, "decoded_token": ".\n\n"}, "1294": {"logprob": -3.887425422668457, "rank": 4, "decoded_token": " in"}, "2425": {"logprob": -4.012425422668457, "rank": 5, "decoded_token": " under"}}, {"30594": {"logprob": -1.8202354907989502, "rank": 1, "decoded_token": " distant"}, "23745": {"logprob": -2.13273549079895, "rank": 2, "decoded_token": " snow"}, "27469": {"logprob": -2.25773549079895, "rank": 3, "decoded_token": " peaks"}, "11223": {"logprob": -2.94523549079895, "rank": 4, "decoded_token": " green"}, "11692": {"logprob": -3.07023549079895, "rank": 5, "decoded_token": " mist"}}, {"27469": {"logprob": -0.40320226550102234, "rank": 1, "decoded_token": " peaks"}, "23745": {"logprob": -2.2157022953033447, "rank": 2, "decoded_token": " snow"}, "35463": {"logprob": -3.4032022953033447, "rank": 3, "decoded_token": " mountains"}, "24361": {"logprob": -3.4032022953033447, "rank": 4, "decoded_token": " mountain"}, "46866": {"logprob": -4.090702056884766, "rank": 5, "decoded_token": " clouds"}}, {"1338": {"logprob": -0.5224027633666992, "rank": 1, "decoded_token": ".\n\n"}, "1626": {"logprob": -1.3349027633666992, "rank": 2, "decoded_token": ".\n"}, "2425": {"logprob": -3.397402763366699, "rank": 3, "decoded_token": " under"}, "1294": {"logprob": -4.084902763366699, "rank": 4, "decoded_token": " in"}, "13875": {"logprob": -4.334902763366699, "rank": 5, "decoded_token": " covered"}}, {"1050": {"logprob": -0.6029570698738098, "rank": 1, "decoded_token": "2"}, "1256": {"logprob": -1.352957010269165, "rank": 2, "decoded_token": " "}, "11745": {"logprob": -3.227957010269165, "rank": 3, "decoded_token": "Here"}, "1293": {"logprob": -3.915457010269165, "rank": 4, "decoded_token": " "}, "4393": {"logprob": -4.602957248687744, "rank": 5, "decoded_token": "For"}}, {"1046": {"logprob": -0.19250261783599854, "rank": 1, "decoded_token": "."}, "1319": {"logprob": -3.567502498626709, "rank": 2, "decoded_token": " ("}, "1626": {"logprob": -4.505002498626709, "rank": 3, "decoded_token": ".\n"}, "1877": {"logprob": -5.942502498626709, "rank": 4, "decoded_token": ":\n"}, "26667": {"logprob": -6.442502498626709, "rank": 5, "decoded_token": ".("}}, {"1531": {"logprob": -1.897325873374939, "rank": 1, "decoded_token": " The"}, "9380": {"logprob": -1.897325873374939, "rank": 2, "decoded_token": " Here"}, "1319": {"logprob": -2.0848259925842285, "rank": 3, "decoded_token": " ("}, "2898": {"logprob": -2.8973259925842285, "rank": 4, "decoded_token": " For"}, "10322": {"logprob": -3.3348259925842285, "rank": 5, "decoded_token": " Second"}}, {"5888": {"logprob": -1.483720064163208, "rank": 1, "decoded_token": " third"}, "2667": {"logprob": -2.296220064163208, "rank": 2, "decoded_token": " second"}, "5662": {"logprob": -2.858720064163208, "rank": 3, "decoded_token": " provided"}, "3937": {"logprob": -3.296220064163208, "rank": 4, "decoded_token": " image"}, "6298": {"logprob": -3.421220064163208, "rank": 5, "decoded_token": " correct"}}, {"3937": {"logprob": -0.6041796207427979, "rank": 1, "decoded_token": " image"}, "1319": {"logprob": -2.604179620742798, "rank": 2, "decoded_token": " ("}, "2016": {"logprob": -2.729179620742798, "rank": 3, "decoded_token": " set"}, "1321": {"logprob": -3.916679620742798, "rank": 4, "decoded_token": " and"}, "13083": {"logprob": -3.979179620742798, "rank": 5, "decoded_token": " picture"}}, {"51948": {"logprob": -1.6496541500091553, "rank": 1, "decoded_token": " depicts"}, "1319": {"logprob": -1.8371541500091553, "rank": 2, "decoded_token": " ("}, "1058": {"logprob": -1.9621541500091553, "rank": 3, "decoded_token": ":"}, "25981": {"logprob": -2.4621541500091553, "rank": 4, "decoded_token": " displays"}, "1395": {"logprob": -2.7746541500091553, "rank": 5, "decoded_token": " is"}}, {"1261": {"logprob": -0.5954864025115967, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -2.5954864025115967, "rank": 2, "decoded_token": " an"}, "22140": {"logprob": -2.6579864025115967, "rank": 3, "decoded_token": " waves"}, "1278": {"logprob": -2.9704864025115967, "rank": 4, "decoded_token": " the"}, "26517": {"logprob": -3.9079864025115967, "rank": 5, "decoded_token": " calm"}}, {"2169": {"logprob": -2.0800817012786865, "rank": 1, "decoded_token": " ser"}, "10726": {"logprob": -2.2050817012786865, "rank": 2, "decoded_token": " scen"}, "29397": {"logprob": -2.3925817012786865, "rank": 3, "decoded_token": " beach"}, "2965": {"logprob": -2.4550817012786865, "rank": 4, "decoded_token": " person"}, "1958": {"logprob": -2.8925817012786865, "rank": 5, "decoded_token": " sur"}}, {"2509": {"logprob": -0.0010151476599276066, "rank": 1, "decoded_token": "ene"}, "1391": {"logprob": -8.188514709472656, "rank": 2, "decoded_token": "if"}, "25863": {"logprob": -8.938514709472656, "rank": 3, "decoded_token": "rated"}, "3414": {"logprob": -9.438514709472656, "rank": 4, "decoded_token": "ena"}, "10049": {"logprob": -9.688514709472656, "rank": 5, "decoded_token": "\u00e8ne"}}, {"29397": {"logprob": -0.5060035586357117, "rank": 1, "decoded_token": " beach"}, "62557": {"logprob": -2.5685036182403564, "rank": 2, "decoded_token": " seas"}, "27208": {"logprob": -2.9435036182403564, "rank": 3, "decoded_token": " ocean"}, "38167": {"logprob": -3.0685036182403564, "rank": 4, "decoded_token": " coastal"}, "13327": {"logprob": -3.2560036182403564, "rank": 5, "decoded_token": " scene"}}, {"13327": {"logprob": -0.8083749413490295, "rank": 1, "decoded_token": " scene"}, "1454": {"logprob": -1.1208748817443848, "rank": 2, "decoded_token": " with"}, "1513": {"logprob": -2.9333748817443848, "rank": 3, "decoded_token": " at"}, "3184": {"logprob": -3.4958748817443848, "rank": 4, "decoded_token": " during"}, "2478": {"logprob": -4.058374881744385, "rank": 5, "decoded_token": " where"}}, {"1454": {"logprob": -0.322070449590683, "rank": 1, "decoded_token": " with"}, "3184": {"logprob": -2.447070360183716, "rank": 2, "decoded_token": " during"}, "1513": {"logprob": -2.759570360183716, "rank": 3, "decoded_token": " at"}, "2425": {"logprob": -3.509570360183716, "rank": 4, "decoded_token": " under"}, "2478": {"logprob": -3.697070360183716, "rank": 5, "decoded_token": " where"}}, {"26905": {"logprob": -1.1276788711547852, "rank": 1, "decoded_token": " gentle"}, "22140": {"logprob": -1.4401788711547852, "rank": 2, "decoded_token": " waves"}, "1261": {"logprob": -2.377678871154785, "rank": 3, "decoded_token": " a"}, "3306": {"logprob": -2.815178871154785, "rank": 4, "decoded_token": " people"}, "3709": {"logprob": -3.252678871154785, "rank": 5, "decoded_token": " small"}}, {"22140": {"logprob": -0.048392221331596375, "rank": 1, "decoded_token": " waves"}, "27208": {"logprob": -3.6733922958374023, "rank": 2, "decoded_token": " ocean"}, "29661": {"logprob": -5.735892295837402, "rank": 3, "decoded_token": " surf"}, "32911": {"logprob": -5.735892295837402, "rank": 4, "decoded_token": " rolling"}, "11196": {"logprob": -6.173392295837402, "rank": 5, "decoded_token": " sea"}}, {"11981": {"logprob": -1.4009065628051758, "rank": 1, "decoded_token": " meeting"}, "1321": {"logprob": -1.5884065628051758, "rank": 2, "decoded_token": " and"}, "86928": {"logprob": -2.088406562805176, "rank": 3, "decoded_token": " crashing"}, "1427": {"logprob": -2.650906562805176, "rank": 4, "decoded_token": " la"}, "44278": {"logprob": -2.838406562805176, "rank": 5, "decoded_token": " approaching"}}, {"1278": {"logprob": -0.8703328967094421, "rank": 1, "decoded_token": " the"}, "100991": {"logprob": -1.370332956314087, "rank": 2, "decoded_token": " sandy"}, "1261": {"logprob": -1.932832956314087, "rank": 3, "decoded_token": " a"}, "14693": {"logprob": -2.932832956314087, "rank": 4, "decoded_token": " sand"}, "46422": {"logprob": -3.557832956314087, "rank": 5, "decoded_token": " shore"}}, {"46422": {"logprob": -0.19113700091838837, "rank": 1, "decoded_token": " shore"}, "100991": {"logprob": -2.7536370754241943, "rank": 2, "decoded_token": " sandy"}, "1627": {"logprob": -3.0036370754241943, "rank": 3, "decoded_token": " sh"}, "14693": {"logprob": -3.3786370754241943, "rank": 4, "decoded_token": " sand"}, "124562": {"logprob": -5.378636837005615, "rank": 5, "decoded_token": " coastline"}}, {"1321": {"logprob": -1.191644549369812, "rank": 1, "decoded_token": " and"}, "3184": {"logprob": -2.0666446685791016, "rank": 2, "decoded_token": " during"}, "1338": {"logprob": -2.3166446685791016, "rank": 3, "decoded_token": ".\n\n"}, "1044": {"logprob": -2.3791446685791016, "rank": 4, "decoded_token": ","}, "1513": {"logprob": -2.5666446685791016, "rank": 5, "decoded_token": " at"}}, {"1261": {"logprob": -0.937598705291748, "rank": 1, "decoded_token": " a"}, "3306": {"logprob": -2.000098705291748, "rank": 2, "decoded_token": " people"}, "29661": {"logprob": -2.625098705291748, "rank": 3, "decoded_token": " surf"}, "2269": {"logprob": -2.937598705291748, "rank": 4, "decoded_token": " some"}, "30594": {"logprob": -3.062598705291748, "rank": 5, "decoded_token": " distant"}}, {"92731": {"logprob": -1.4627695083618164, "rank": 1, "decoded_token": " lone"}, "2965": {"logprob": -1.6502695083618164, "rank": 2, "decoded_token": " person"}, "4517": {"logprob": -2.0877695083618164, "rank": 3, "decoded_token": " few"}, "81249": {"logprob": -2.3377695083618164, "rank": 4, "decoded_token": " silhouette"}, "79013": {"logprob": -2.4627695083618164, "rank": 5, "decoded_token": " solitary"}}, {"2965": {"logprob": -0.9964981079101562, "rank": 1, "decoded_token": " person"}, "8240": {"logprob": -1.1839981079101562, "rank": 2, "decoded_token": " figure"}, "1958": {"logprob": -1.9964981079101562, "rank": 3, "decoded_token": " sur"}, "81249": {"logprob": -3.2464981079101562, "rank": 4, "decoded_token": " silhouette"}, "4597": {"logprob": -3.3089981079101562, "rank": 5, "decoded_token": " individual"}}, {"19710": {"logprob": -1.8438996076583862, "rank": 1, "decoded_token": " walking"}, "15866": {"logprob": -2.093899726867676, "rank": 2, "decoded_token": " standing"}, "6117": {"logprob": -2.093899726867676, "rank": 3, "decoded_token": " near"}, "1285": {"logprob": -2.468899726867676, "rank": 4, "decoded_token": " w"}, "1294": {"logprob": -2.843899726867676, "rank": 5, "decoded_token": " in"}}, {"4837": {"logprob": -1.405532717704773, "rank": 1, "decoded_token": " along"}, "6117": {"logprob": -1.718032717704773, "rank": 2, "decoded_token": " near"}, "1408": {"logprob": -2.5930328369140625, "rank": 3, "decoded_token": " on"}, "8994": {"logprob": -2.7180328369140625, "rank": 4, "decoded_token": " towards"}, "1338": {"logprob": -2.8430328369140625, "rank": 5, "decoded_token": ".\n\n"}}, {"1278": {"logprob": -0.1722739040851593, "rank": 1, "decoded_token": " the"}, "1494": {"logprob": -2.734773874282837, "rank": 2, "decoded_token": " it"}, "1338": {"logprob": -3.484773874282837, "rank": 3, "decoded_token": ".\n\n"}, "1261": {"logprob": -3.984773874282837, "rank": 4, "decoded_token": " a"}, "1626": {"logprob": -4.984774112701416, "rank": 5, "decoded_token": ".\n"}}, {"46422": {"logprob": -1.466295599937439, "rank": 1, "decoded_token": " shore"}, "1627": {"logprob": -1.528795599937439, "rank": 2, "decoded_token": " sh"}, "4180": {"logprob": -1.841295599937439, "rank": 3, "decoded_token": " water"}, "10314": {"logprob": -2.0912957191467285, "rank": 4, "decoded_token": " edge"}, "14693": {"logprob": -3.0912957191467285, "rank": 5, "decoded_token": " sand"}}, {"1338": {"logprob": -0.9661335945129395, "rank": 1, "decoded_token": ".\n\n"}, "2839": {"logprob": -2.0911335945129395, "rank": 2, "decoded_token": "line"}, "1626": {"logprob": -2.1536335945129395, "rank": 3, "decoded_token": ".\n"}, "1513": {"logprob": -2.1536335945129395, "rank": 4, "decoded_token": " at"}, "3184": {"logprob": -2.6536335945129395, "rank": 5, "decoded_token": " during"}}, {"1051": {"logprob": -0.8673074841499329, "rank": 1, "decoded_token": "3"}, "1256": {"logprob": -2.054807424545288, "rank": 2, "decoded_token": " "}, "11745": {"logprob": -2.242307424545288, "rank": 3, "decoded_token": "Here"}, "1052": {"logprob": -2.242307424545288, "rank": 4, "decoded_token": "4"}, "1050": {"logprob": -3.367307424545288, "rank": 5, "decoded_token": "2"}}, {"1046": {"logprob": -0.09165985882282257, "rank": 1, "decoded_token": "."}, "1626": {"logprob": -4.341660022735596, "rank": 2, "decoded_token": ".\n"}, "1319": {"logprob": -4.841660022735596, "rank": 3, "decoded_token": " ("}, "1045": {"logprob": -6.029160022735596, "rank": 4, "decoded_token": "-"}, "1321": {"logprob": -6.529160022735596, "rank": 5, "decoded_token": " and"}}, {"1531": {"logprob": -0.5217734575271606, "rank": 1, "decoded_token": " The"}, "9380": {"logprob": -2.396773338317871, "rank": 2, "decoded_token": " Here"}, "2898": {"logprob": -3.521773338317871, "rank": 3, "decoded_token": " For"}, "45663": {"logprob": -3.959273338317871, "rank": 4, "decoded_token": " Fourth"}, "1319": {"logprob": -4.084273338317871, "rank": 5, "decoded_token": " ("}}, {"12432": {"logprob": -0.41262897849082947, "rank": 1, "decoded_token": " fourth"}, "5662": {"logprob": -3.8501288890838623, "rank": 2, "decoded_token": " provided"}, "5888": {"logprob": -4.037629127502441, "rank": 3, "decoded_token": " third"}, "3937": {"logprob": -4.037629127502441, "rank": 4, "decoded_token": " image"}, "2667": {"logprob": -4.162629127502441, "rank": 5, "decoded_token": " second"}}, {"3937": {"logprob": -0.5903608798980713, "rank": 1, "decoded_token": " image"}, "1319": {"logprob": -2.3403608798980713, "rank": 2, "decoded_token": " ("}, "2016": {"logprob": -2.7153608798980713, "rank": 3, "decoded_token": " set"}, "1321": {"logprob": -3.4653608798980713, "rank": 4, "decoded_token": " and"}, "1925": {"logprob": -4.090360641479492, "rank": 5, "decoded_token": " one"}}, {"6971": {"logprob": -1.8004755973815918, "rank": 1, "decoded_token": " features"}, "1319": {"logprob": -2.175475597381592, "rank": 2, "decoded_token": " ("}, "6122": {"logprob": -2.300475597381592, "rank": 3, "decoded_token": " shows"}, "1395": {"logprob": -2.487975597381592, "rank": 4, "decoded_token": " is"}, "1058": {"logprob": -2.487975597381592, "rank": 5, "decoded_token": ":"}}, {"1261": {"logprob": -0.16380631923675537, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -2.976306438446045, "rank": 2, "decoded_token": " an"}, "2295": {"logprob": -3.351306438446045, "rank": 3, "decoded_token": " two"}, "1278": {"logprob": -4.413806438446045, "rank": 4, "decoded_token": " the"}, "16429": {"logprob": -4.851306438446045, "rank": 5, "decoded_token": " trees"}}, {"53301": {"logprob": -1.6735851764678955, "rank": 1, "decoded_token": " winding"}, "59396": {"logprob": -1.9235851764678955, "rank": 2, "decoded_token": " gravel"}, "101727": {"logprob": -2.1110851764678955, "rank": 3, "decoded_token": " paved"}, "47945": {"logprob": -3.1110851764678955, "rank": 4, "decoded_token": " dirt"}, "23874": {"logprob": -3.1110851764678955, "rank": 5, "decoded_token": " pictures"}}, {"59396": {"logprob": -0.6121145486831665, "rank": 1, "decoded_token": " gravel"}, "47945": {"logprob": -1.7996145486831665, "rank": 2, "decoded_token": " dirt"}, "101727": {"logprob": -2.612114429473877, "rank": 3, "decoded_token": " paved"}, "3549": {"logprob": -2.799614429473877, "rank": 4, "decoded_token": " path"}, "1044": {"logprob": -2.862114429473877, "rank": 5, "decoded_token": ","}}, {"3549": {"logprob": -0.4002755880355835, "rank": 1, "decoded_token": " path"}, "14801": {"logprob": -1.7752755880355835, "rank": 2, "decoded_token": " pathway"}, "1505": {"logprob": -2.275275707244873, "rank": 3, "decoded_token": " or"}, "33659": {"logprob": -3.962775707244873, "rank": 4, "decoded_token": " trail"}, "9480": {"logprob": -4.337775707244873, "rank": 5, "decoded_token": " road"}}, {"121040": {"logprob": -1.1325652599334717, "rank": 1, "decoded_token": " bordered"}, "121313": {"logprob": -1.6950652599334717, "rank": 2, "decoded_token": " flanked"}, "29817": {"logprob": -2.1950652599334717, "rank": 3, "decoded_token": " surrounded"}, "8924": {"logprob": -2.6950652599334717, "rank": 4, "decoded_token": " leading"}, "1294": {"logprob": -2.8200652599334717, "rank": 5, "decoded_token": " in"}}, {"1536": {"logprob": -0.02329995296895504, "rank": 1, "decoded_token": " by"}, "1454": {"logprob": -4.5233001708984375, "rank": 2, "decoded_token": " with"}, "1408": {"logprob": -4.5858001708984375, "rank": 3, "decoded_token": " on"}, "98245": {"logprob": -8.460800170898438, "rank": 4, "decoded_token": " beautifully"}, "100910": {"logprob": -8.648300170898438, "rank": 5, "decoded_token": " neatly"}}, {"23170": {"logprob": -1.2441778182983398, "rank": 1, "decoded_token": " grass"}, "1295": {"logprob": -1.4941778182983398, "rank": 2, "decoded_token": " l"}, "11223": {"logprob": -1.5566778182983398, "rank": 3, "decoded_token": " green"}, "1261": {"logprob": -3.05667781829834, "rank": 4, "decoded_token": " a"}, "4174": {"logprob": -3.61917781829834, "rank": 5, "decoded_token": " gre"}}, {"1321": {"logprob": -0.8162240982055664, "rank": 1, "decoded_token": " and"}, "1121": {"logprob": -1.4412240982055664, "rank": 2, "decoded_token": "y"}, "1044": {"logprob": -1.8787240982055664, "rank": 3, "decoded_token": ","}, "8924": {"logprob": -3.3162240982055664, "rank": 4, "decoded_token": " leading"}, "1454": {"logprob": -3.5037240982055664, "rank": 5, "decoded_token": " with"}}, {"16429": {"logprob": -1.8316444158554077, "rank": 1, "decoded_token": " trees"}, "87833": {"logprob": -2.4566445350646973, "rank": 2, "decoded_token": " flowering"}, "17744": {"logprob": -2.5816445350646973, "rank": 3, "decoded_token": " blo"}, "1261": {"logprob": -2.6441445350646973, "rank": 4, "decoded_token": " a"}, "29817": {"logprob": -3.0191445350646973, "rank": 5, "decoded_token": " surrounded"}}, {"1294": {"logprob": -1.625852108001709, "rank": 1, "decoded_token": " in"}, "1044": {"logprob": -1.750852108001709, "rank": 2, "decoded_token": ","}, "2425": {"logprob": -2.125852108001709, "rank": 3, "decoded_token": " under"}, "8924": {"logprob": -2.250852108001709, "rank": 4, "decoded_token": " leading"}, "1408": {"logprob": -2.625852108001709, "rank": 5, "decoded_token": " on"}}, {"1261": {"logprob": -0.3194517195224762, "rank": 1, "decoded_token": " a"}, "1278": {"logprob": -2.3194518089294434, "rank": 2, "decoded_token": " the"}, "1420": {"logprob": -2.5694518089294434, "rank": 3, "decoded_token": " an"}, "2549": {"logprob": -2.6319518089294434, "rank": 4, "decoded_token": " what"}, "5346": {"logprob": -5.569451808929443, "rank": 5, "decoded_token": " front"}}, {"23874": {"logprob": -1.2531365156173706, "rank": 1, "decoded_token": " pictures"}, "10726": {"logprob": -1.6906365156173706, "rank": 2, "decoded_token": " scen"}, "54742": {"logprob": -2.12813663482666, "rank": 3, "decoded_token": " peaceful"}, "12097": {"logprob": -2.69063663482666, "rank": 4, "decoded_token": " park"}, "2169": {"logprob": -3.00313663482666, "rank": 5, "decoded_token": " ser"}}, {"1872": {"logprob": -0.0001784403866622597, "rank": 1, "decoded_token": "que"}, "1348": {"logprob": -8.875178337097168, "rank": 2, "decoded_token": "qu"}, "21451": {"logprob": -11.937678337097168, "rank": 3, "decoded_token": "QUE"}, "18954": {"logprob": -12.687678337097168, "rank": 4, "decoded_token": "qu\u00e9"}, "14016": {"logprob": -12.812678337097168, "rank": 5, "decoded_token": "quare"}}, {"41730": {"logprob": -1.2438380718231201, "rank": 1, "decoded_token": " outdoor"}, "12097": {"logprob": -1.4938380718231201, "rank": 2, "decoded_token": " park"}, "1044": {"logprob": -1.8688380718231201, "rank": 3, "decoded_token": ","}, "28035": {"logprob": -2.55633807182312, "rank": 4, "decoded_token": " landscape"}, "6967": {"logprob": -2.68133807182312, "rank": 5, "decoded_token": " natural"}}, {"9436": {"logprob": -0.28005897998809814, "rank": 1, "decoded_token": " setting"}, "6093": {"logprob": -2.9050588607788086, "rank": 2, "decoded_token": " environment"}, "12097": {"logprob": -2.9675588607788086, "rank": 3, "decoded_token": " park"}, "28035": {"logprob": -3.5300588607788086, "rank": 4, "decoded_token": " landscape"}, "4457": {"logprob": -3.7175588607788086, "rank": 5, "decoded_token": " area"}}, {"1338": {"logprob": -0.40908101201057434, "rank": 1, "decoded_token": ".\n\n"}, "1626": {"logprob": -2.284080982208252, "rank": 2, "decoded_token": ".\n"}, "2425": {"logprob": -2.721580982208252, "rank": 3, "decoded_token": " under"}, "1454": {"logprob": -3.096580982208252, "rank": 4, "decoded_token": " with"}, "1044": {"logprob": -3.284080982208252, "rank": 5, "decoded_token": ","}}, {"1052": {"logprob": -1.0536651611328125, "rank": 1, "decoded_token": "4"}, "11745": {"logprob": -2.1161651611328125, "rank": 2, "decoded_token": "Here"}, "16840": {"logprob": -3.2411651611328125, "rank": 3, "decoded_token": "Since"}, "2892": {"logprob": -3.3661651611328125, "rank": 4, "decoded_token": "To"}, "1040": {"logprob": -3.3661651611328125, "rank": 5, "decoded_token": "("}}, {"1046": {"logprob": -0.15314939618110657, "rank": 1, "decoded_token": "."}, "1626": {"logprob": -4.215649604797363, "rank": 2, "decoded_token": ".\n"}, "1319": {"logprob": -4.528149604797363, "rank": 3, "decoded_token": " ("}, "26667": {"logprob": -5.840649604797363, "rank": 4, "decoded_token": ".("}, "9380": {"logprob": -6.340649604797363, "rank": 5, "decoded_token": " Here"}}, {"1531": {"logprob": -1.7034302949905396, "rank": 1, "decoded_token": " The"}, "9380": {"logprob": -2.20343017578125, "rank": 2, "decoded_token": " Here"}, "1319": {"logprob": -2.26593017578125, "rank": 3, "decoded_token": " ("}, "2898": {"logprob": -2.95343017578125, "rank": 4, "decoded_token": " For"}, "9748": {"logprob": -3.01593017578125, "rank": 5, "decoded_token": " Since"}}, {"19723": {"logprob": -1.0940117835998535, "rank": 1, "decoded_token": " fifth"}, "5662": {"logprob": -3.2190117835998535, "rank": 2, "decoded_token": " provided"}, "3804": {"logprob": -3.4690117835998535, "rank": 3, "decoded_token": " last"}, "12432": {"logprob": -3.5940117835998535, "rank": 4, "decoded_token": " fourth"}, "5719": {"logprob": -3.6565117835998535, "rank": 5, "decoded_token": " initial"}}, {"3937": {"logprob": -1.0828688144683838, "rank": 1, "decoded_token": " image"}, "1319": {"logprob": -2.020368814468384, "rank": 2, "decoded_token": " ("}, "1925": {"logprob": -2.332868814468384, "rank": 3, "decoded_token": " one"}, "2016": {"logprob": -3.457868814468384, "rank": 4, "decoded_token": " set"}, "5662": {"logprob": -3.895368814468384, "rank": 5, "decoded_token": " provided"}}, {"1319": {"logprob": -0.9146400094032288, "rank": 1, "decoded_token": " ("}, "10045": {"logprob": -2.789639949798584, "rank": 2, "decoded_token": " isn"}, "1395": {"logprob": -2.914639949798584, "rank": 3, "decoded_token": " is"}, "5662": {"logprob": -3.477139949798584, "rank": 4, "decoded_token": " provided"}, "1294": {"logprob": -3.602139949798584, "rank": 5, "decoded_token": " in"}}, {"3715": {"logprob": -2.2420246601104736, "rank": 1, "decoded_token": "not"}, "3265": {"logprob": -3.3045246601104736, "rank": 2, "decoded_token": "the"}, "11018": {"logprob": -3.3670246601104736, "rank": 3, "decoded_token": "which"}, "1391": {"logprob": -3.4920246601104736, "rank": 4, "decoded_token": "if"}, "5011": {"logprob": -3.5545246601104736, "rank": 5, "decoded_token": "from"}}, {"24512": {"logprob": -1.8148819208145142, "rank": 1, "decoded_token": " applicable"}, "4326": {"logprob": -2.6273818016052246, "rank": 2, "decoded_token": " shown"}, "5662": {"logprob": -2.8773818016052246, "rank": 3, "decoded_token": " provided"}, "5656": {"logprob": -3.1898818016052246, "rank": 4, "decoded_token": " included"}, "1805": {"logprob": -3.1898818016052246, "rank": 5, "decoded_token": " part"}}, {"1435": {"logprob": -1.9905282258987427, "rank": 1, "decoded_token": " as"}, "1317": {"logprob": -2.115528106689453, "rank": 2, "decoded_token": " to"}, "1294": {"logprob": -2.303028106689453, "rank": 3, "decoded_token": " in"}, "3226": {"logprob": -2.553028106689453, "rank": 4, "decoded_token": " here"}, "1394": {"logprob": -2.553028106689453, "rank": 5, "decoded_token": " for"}}, {"1651": {"logprob": -1.3723630905151367, "rank": 1, "decoded_token": " per"}, "1494": {"logprob": -1.6848630905151367, "rank": 2, "decoded_token": " it"}, "2156": {"logprob": -2.4348630905151367, "rank": 3, "decoded_token": " there"}, "1278": {"logprob": -2.9973630905151367, "rank": 4, "decoded_token": " the"}, "1636": {"logprob": -3.0598630905151367, "rank": 5, "decoded_token": " you"}}, {"1278": {"logprob": -1.4684513807296753, "rank": 1, "decoded_token": " the"}, "2143": {"logprob": -1.5309513807296753, "rank": 2, "decoded_token": " your"}, "5719": {"logprob": -2.218451499938965, "rank": 3, "decoded_token": " initial"}, "5662": {"logprob": -3.280951499938965, "rank": 4, "decoded_token": " provided"}, "4618": {"logprob": -3.405951499938965, "rank": 5, "decoded_token": " original"}}, {"5719": {"logprob": -1.7317447662353516, "rank": 1, "decoded_token": " initial"}, "5662": {"logprob": -2.4817447662353516, "rank": 2, "decoded_token": " provided"}, "4618": {"logprob": -2.6692447662353516, "rank": 3, "decoded_token": " original"}, "2158": {"logprob": -3.1067447662353516, "rank": 4, "decoded_token": " first"}, "4265": {"logprob": -3.2317447662353516, "rank": 5, "decoded_token": " given"}}, {"4546": {"logprob": -2.323227643966675, "rank": 1, "decoded_token": " request"}, "4098": {"logprob": -2.635727643966675, "rank": 2, "decoded_token": " question"}, "5662": {"logprob": -2.760727643966675, "rank": 3, "decoded_token": " provided"}, "6360": {"logprob": -2.885727643966675, "rank": 4, "decoded_token": " description"}, "4249": {"logprob": -3.260727643966675, "rank": 5, "decoded_token": " single"}}, {"1041": {"logprob": -2.016270160675049, "rank": 1, "decoded_token": ")"}, "1394": {"logprob": -2.203770160675049, "rank": 2, "decoded_token": " for"}, "1044": {"logprob": -2.891270160675049, "rank": 3, "decoded_token": ","}, "3640": {"logprob": -3.016270160675049, "rank": 4, "decoded_token": "):\n"}, "1321": {"logprob": -3.078770160675049, "rank": 5, "decoded_token": " and"}}, {"2168": {"logprob": -2.4032278060913086, "rank": 1, "decoded_token": " would"}, "1395": {"logprob": -2.4657278060913086, "rank": 2, "decoded_token": " is"}, "7444": {"logprob": -2.5907278060913086, "rank": 3, "decoded_token": " seems"}, "2715": {"logprob": -2.6532278060913086, "rank": 4, "decoded_token": " should"}, "10045": {"logprob": -3.2157278060913086, "rank": 5, "decoded_token": " isn"}}, {"1402": {"logprob": -1.638601303100586, "rank": 1, "decoded_token": " be"}, "1605": {"logprob": -2.138601303100586, "rank": 2, "decoded_token": " not"}, "96594": {"logprob": -2.513601303100586, "rank": 3, "decoded_token": " logically"}, "2534": {"logprob": -2.888601303100586, "rank": 4, "decoded_token": " need"}, "79961": {"logprob": -3.201101303100586, "rank": 5, "decoded_token": " ideally"}}, {"1278": {"logprob": -2.479240894317627, "rank": 1, "decoded_token": " the"}, "1261": {"logprob": -2.854240894317627, "rank": 2, "decoded_token": " a"}, "3866": {"logprob": -3.041740894317627, "rank": 3, "decoded_token": " another"}, "57773": {"logprob": -3.479240894317627, "rank": 4, "decoded_token": " irrelevant"}, "1877": {"logprob": -3.604240894317627, "rank": 5, "decoded_token": ":\n"}}, {"1925": {"logprob": -2.5215530395507812, "rank": 1, "decoded_token": " one"}, "2879": {"logprob": -3.1465530395507812, "rank": 2, "decoded_token": " same"}, "25342": {"logprob": -3.4590530395507812, "rank": 3, "decoded_token": " incorrect"}, "7244": {"logprob": -3.5840530395507812, "rank": 4, "decoded_token": " black"}, "4275": {"logprob": -3.5840530395507812, "rank": 5, "decoded_token": " next"}}, {"1454": {"logprob": -2.0894155502319336, "rank": 1, "decoded_token": " with"}, "1636": {"logprob": -3.0269155502319336, "rank": 2, "decoded_token": " you"}, "1307": {"logprob": -3.0269155502319336, "rank": 3, "decoded_token": " of"}, "1455": {"logprob": -3.2769155502319336, "rank": 4, "decoded_token": " that"}, "1562": {"logprob": -3.4019155502319336, "rank": 5, "decoded_token": " from"}}, {"1278": {"logprob": -1.3837254047393799, "rank": 1, "decoded_token": " the"}, "1261": {"logprob": -1.6962254047393799, "rank": 2, "decoded_token": " a"}, "3866": {"logprob": -3.13372540473938, "rank": 3, "decoded_token": " another"}, "1420": {"logprob": -3.19622540473938, "rank": 4, "decoded_token": " an"}, "2586": {"logprob": -3.69622540473938, "rank": 5, "decoded_token": " just"}}, {"10575": {"logprob": -1.6755551099777222, "rank": 1, "decoded_token": " dog"}, "7244": {"logprob": -2.3005552291870117, "rank": 2, "decoded_token": " black"}, "25342": {"logprob": -3.7380552291870117, "rank": 3, "decoded_token": " incorrect"}, "3549": {"logprob": -3.8630552291870117, "rank": 4, "decoded_token": " path"}, "4811": {"logprob": -4.238055229187012, "rank": 5, "decoded_token": " specific"}}, {"2790": {"logprob": -1.985573649406433, "rank": 1, "decoded_token": " again"}, "1044": {"logprob": -2.4230737686157227, "rank": 2, "decoded_token": ","}, "1562": {"logprob": -2.7355737686157227, "rank": 3, "decoded_token": " from"}, "1338": {"logprob": -2.7980737686157227, "rank": 4, "decoded_token": ".\n\n"}, "1408": {"logprob": -3.0480737686157227, "rank": 5, "decoded_token": " on"}}, {"1044": {"logprob": -1.7941354513168335, "rank": 1, "decoded_token": ","}, "1693": {"logprob": -2.231635570526123, "rank": 2, "decoded_token": " if"}, "1809": {"logprob": -2.731635570526123, "rank": 3, "decoded_token": " but"}, "1505": {"logprob": -3.356635570526123, "rank": 4, "decoded_token": " or"}, "1562": {"logprob": -3.419135570526123, "rank": 5, "decoded_token": " from"}}, {"1809": {"logprob": -1.9572563171386719, "rank": 1, "decoded_token": " but"}, "1878": {"logprob": -2.269756317138672, "rank": 2, "decoded_token": " so"}, "1799": {"logprob": -2.769756317138672, "rank": 3, "decoded_token": " which"}, "1321": {"logprob": -3.769756317138672, "rank": 4, "decoded_token": " and"}, "1693": {"logprob": -3.957256317138672, "rank": 5, "decoded_token": " if"}}, {"4136": {"logprob": -2.412797212600708, "rank": 1, "decoded_token": " since"}, "3226": {"logprob": -2.475297212600708, "rank": 2, "decoded_token": " here"}, "1278": {"logprob": -2.850297212600708, "rank": 3, "decoded_token": " the"}, "2878": {"logprob": -2.975297212600708, "rank": 4, "decoded_token": " let"}, "1494": {"logprob": -3.287797212600708, "rank": 5, "decoded_token": " it"}}, {"1494": {"logprob": -1.4206243753433228, "rank": 1, "decoded_token": " it"}, "1278": {"logprob": -2.045624256134033, "rank": 2, "decoded_token": " the"}, "1729": {"logprob": -2.295624256134033, "rank": 3, "decoded_token": " we"}, "2156": {"logprob": -2.483124256134033, "rank": 4, "decoded_token": " there"}, "1636": {"logprob": -2.483124256134033, "rank": 5, "decoded_token": " you"}}, {"1681": {"logprob": -1.4698597192764282, "rank": 1, "decoded_token": "'s"}, "2190": {"logprob": -1.6573597192764282, "rank": 2, "decoded_token": "\u2019s"}, "1395": {"logprob": -2.1573596000671387, "rank": 3, "decoded_token": " is"}, "1486": {"logprob": -2.5323596000671387, "rank": 4, "decoded_token": " was"}, "7444": {"logprob": -3.2823596000671387, "rank": 5, "decoded_token": " seems"}}, {"5314": {"logprob": -1.6933988332748413, "rank": 1, "decoded_token": " already"}, "1605": {"logprob": -2.130898952484131, "rank": 2, "decoded_token": " not"}, "1278": {"logprob": -2.193398952484131, "rank": 3, "decoded_token": " the"}, "2342": {"logprob": -3.318398952484131, "rank": 4, "decoded_token": " only"}, "13578": {"logprob": -3.568398952484131, "rank": 5, "decoded_token": " repeated"}}, {"5055": {"logprob": -1.3677805662155151, "rank": 1, "decoded_token": " described"}, "13875": {"logprob": -1.4927805662155151, "rank": 2, "decoded_token": " covered"}, "24511": {"logprob": -2.6802806854248047, "rank": 3, "decoded_token": " addressed"}, "10910": {"logprob": -2.7427806854248047, "rank": 4, "decoded_token": " mentioned"}, "5656": {"logprob": -3.4302806854248047, "rank": 5, "decoded_token": " included"}}, {"1044": {"logprob": -1.6195299625396729, "rank": 1, "decoded_token": ","}, "2100": {"logprob": -2.057029962539673, "rank": 2, "decoded_token": ":\n\n"}, "1294": {"logprob": -2.057029962539673, "rank": 3, "decoded_token": " in"}, "1877": {"logprob": -2.369529962539673, "rank": 4, "decoded_token": ":\n"}, "15423": {"logprob": -2.869529962539673, "rank": 5, "decoded_token": " initially"}}, {"3226": {"logprob": -1.7415276765823364, "rank": 1, "decoded_token": " here"}, "2878": {"logprob": -1.8665276765823364, "rank": 2, "decoded_token": " let"}, "1362": {"logprob": -2.304027557373047, "rank": 3, "decoded_token": " I"}, "1494": {"logprob": -2.616527557373047, "rank": 4, "decoded_token": " it"}, "1278": {"logprob": -3.179027557373047, "rank": 5, "decoded_token": " the"}}, {"2190": {"logprob": -1.3488249778747559, "rank": 1, "decoded_token": "\u2019s"}, "1681": {"logprob": -1.4113249778747559, "rank": 2, "decoded_token": "'s"}, "1395": {"logprob": -1.5363249778747559, "rank": 3, "decoded_token": " is"}, "1584": {"logprob": -2.473824977874756, "rank": 4, "decoded_token": " are"}, "1494": {"logprob": -4.348824977874756, "rank": 5, "decoded_token": " it"}}, {"1261": {"logprob": -1.4024583101272583, "rank": 1, "decoded_token": " a"}, "1278": {"logprob": -1.9024583101272583, "rank": 2, "decoded_token": " the"}, "3866": {"logprob": -2.1524581909179688, "rank": 3, "decoded_token": " another"}, "1420": {"logprob": -2.5274581909179688, "rank": 4, "decoded_token": " an"}, "1605": {"logprob": -4.152458190917969, "rank": 5, "decoded_token": " not"}}, {"2801": {"logprob": -2.7750463485717773, "rank": 1, "decoded_token": " different"}, "6468": {"logprob": -3.1500463485717773, "rank": 2, "decoded_token": " focus"}, "21788": {"logprob": -3.2750463485717773, "rank": 3, "decoded_token": " repeat"}, "17793": {"logprob": -3.4625463485717773, "rank": 4, "decoded_token": " summary"}, "13426": {"logprob": -3.6500463485717773, "rank": 5, "decoded_token": " brief"}}, {"6468": {"logprob": -2.8590242862701416, "rank": 1, "decoded_token": " focus"}, "1925": {"logprob": -3.4215242862701416, "rank": 2, "decoded_token": " one"}, "19190": {"logprob": -3.9215242862701416, "rank": 3, "decoded_token": " interpretation"}, "6360": {"logprob": -3.9840242862701416, "rank": 4, "decoded_token": " description"}, "3336": {"logprob": -3.9840242862701416, "rank": 5, "decoded_token": " example"}}, {"1693": {"logprob": -2.391437530517578, "rank": 1, "decoded_token": " if"}, "1877": {"logprob": -2.516437530517578, "rank": 2, "decoded_token": ":\n"}, "1408": {"logprob": -2.516437530517578, "rank": 3, "decoded_token": " on"}, "1058": {"logprob": -2.578937530517578, "rank": 4, "decoded_token": ":"}, "2100": {"logprob": -2.703937530517578, "rank": 5, "decoded_token": ":\n\n"}}, {"1729": {"logprob": -2.6488425731658936, "rank": 1, "decoded_token": " we"}, "6618": {"logprob": -2.6488425731658936, "rank": 2, "decoded_token": " needed"}, "1494": {"logprob": -2.9613425731658936, "rank": 3, "decoded_token": " it"}, "2258": {"logprob": -3.2113425731658936, "rank": 4, "decoded_token": " any"}, "1636": {"logprob": -3.2738425731658936, "rank": 5, "decoded_token": " you"}}, {"3369": {"logprob": -2.0612776279449463, "rank": 1, "decoded_token": " consider"}, "1722": {"logprob": -2.6862776279449463, "rank": 2, "decoded_token": " were"}, "1880": {"logprob": -3.1237776279449463, "rank": 3, "decoded_token": " had"}, "14649": {"logprob": -3.1862776279449463, "rank": 4, "decoded_token": " assume"}, "55328": {"logprob": -3.2487776279449463, "rank": 5, "decoded_token": " mistaken"}}, {"1278": {"logprob": -1.7770118713378906, "rank": 1, "decoded_token": " the"}, "3866": {"logprob": -2.4645118713378906, "rank": 2, "decoded_token": " another"}, "1261": {"logprob": -2.5895118713378906, "rank": 3, "decoded_token": " a"}, "1494": {"logprob": -3.5270118713378906, "rank": 4, "decoded_token": " it"}, "1420": {"logprob": -3.5895118713378906, "rank": 5, "decoded_token": " an"}}, {"3629": {"logprob": -3.3516223430633545, "rank": 1, "decoded_token": " following"}, "4275": {"logprob": -3.5391223430633545, "rank": 2, "decoded_token": " next"}, "12432": {"logprob": -3.7266223430633545, "rank": 3, "decoded_token": " fourth"}, "5719": {"logprob": -3.7891223430633545, "rank": 4, "decoded_token": " initial"}, "2158": {"logprob": -3.9141223430633545, "rank": 5, "decoded_token": " first"}}, {"75275": {"logprob": -3.0541036128997803, "rank": 1, "decoded_token": " hypothetical"}, "2100": {"logprob": -3.1166036128997803, "rank": 2, "decoded_token": ":\n\n"}, "1877": {"logprob": -3.1166036128997803, "rank": 3, "decoded_token": ":\n"}, "1435": {"logprob": -4.179103851318359, "rank": 4, "decoded_token": " as"}, "1319": {"logprob": -4.179103851318359, "rank": 5, "decoded_token": " ("}}, {"1877": {"logprob": -2.70977520942688, "rank": 1, "decoded_token": ":\n"}, "2100": {"logprob": -2.83477520942688, "rank": 2, "decoded_token": ":\n\n"}, "1115": {"logprob": -3.20977520942688, "rank": 3, "decoded_token": "s"}, "12432": {"logprob": -3.64727520942688, "rank": 4, "decoded_token": " fourth"}, "1058": {"logprob": -3.89727520942688, "rank": 5, "decoded_token": ":"}}, {"1256": {"logprob": -0.4959573745727539, "rank": 1, "decoded_token": " "}, "1293": {"logprob": -1.870957374572754, "rank": 2, "decoded_token": " "}, "11745": {"logprob": -4.183457374572754, "rank": 3, "decoded_token": "Here"}, "1784": {"logprob": -4.495957374572754, "rank": 4, "decoded_token": "The"}, "1053": {"logprob": -4.495957374572754, "rank": 5, "decoded_token": "5"}}, {"1462": {"logprob": -1.5276975631713867, "rank": 1, "decoded_token": " -"}, "1319": {"logprob": -2.4026975631713867, "rank": 2, "decoded_token": " ("}, "9246": {"logprob": -3.2776975631713867, "rank": 3, "decoded_token": " Let"}, "9380": {"logprob": -3.3401975631713867, "rank": 4, "decoded_token": " Here"}, "1531": {"logprob": -3.3401975631713867, "rank": 5, "decoded_token": " The"}}, {"1531": {"logprob": -2.32918381690979, "rank": 1, "decoded_token": " The"}, "1319": {"logprob": -2.82918381690979, "rank": 2, "decoded_token": " ("}, "1349": {"logprob": -3.01668381690979, "rank": 3, "decoded_token": " A"}, "2898": {"logprob": -3.20418381690979, "rank": 4, "decoded_token": " For"}, "9380": {"logprob": -3.57918381690979, "rank": 5, "decoded_token": " Here"}}, {"5719": {"logprob": -2.6479814052581787, "rank": 1, "decoded_token": " initial"}, "19723": {"logprob": -3.2729814052581787, "rank": 2, "decoded_token": " fifth"}, "3804": {"logprob": -3.3979814052581787, "rank": 3, "decoded_token": " last"}, "25342": {"logprob": -3.8979814052581787, "rank": 4, "decoded_token": " incorrect"}, "5662": {"logprob": -3.9604814052581787, "rank": 5, "decoded_token": " provided"}}, {"5662": {"logprob": -2.9346542358398438, "rank": 1, "decoded_token": " provided"}, "6468": {"logprob": -3.1846542358398438, "rank": 2, "decoded_token": " focus"}, "7244": {"logprob": -3.6221542358398438, "rank": 3, "decoded_token": " black"}, "10575": {"logprob": -3.6221542358398438, "rank": 4, "decoded_token": " dog"}, "6360": {"logprob": -3.8096542358398438, "rank": 5, "decoded_token": " description"}}, {"3937": {"logprob": -3.0049667358398438, "rank": 1, "decoded_token": " image"}, "10575": {"logprob": -3.0674667358398438, "rank": 2, "decoded_token": " dog"}, "8061": {"logprob": -3.5049667358398438, "rank": 3, "decoded_token": " images"}, "2016": {"logprob": -3.5049667358398438, "rank": 4, "decoded_token": " set"}, "2667": {"logprob": -3.5674667358398438, "rank": 5, "decoded_token": " second"}}, {"1395": {"logprob": -2.7246885299682617, "rank": 1, "decoded_token": " is"}, "1319": {"logprob": -2.7871885299682617, "rank": 2, "decoded_token": " ("}, "1307": {"logprob": -3.6621885299682617, "rank": 3, "decoded_token": " of"}, "2016": {"logprob": -3.7871885299682617, "rank": 4, "decoded_token": " set"}, "1058": {"logprob": -3.7871885299682617, "rank": 5, "decoded_token": ":"}}, {"1261": {"logprob": -2.914872646331787, "rank": 1, "decoded_token": " a"}, "1605": {"logprob": -3.164872646331787, "rank": 2, "decoded_token": " not"}, "1278": {"logprob": -3.164872646331787, "rank": 3, "decoded_token": " the"}, "2586": {"logprob": -3.227372646331787, "rank": 4, "decoded_token": " just"}, "13578": {"logprob": -3.352372646331787, "rank": 5, "decoded_token": " repeated"}}, {"7244": {"logprob": -2.4122366905212402, "rank": 1, "decoded_token": " black"}, "15115": {"logprob": -3.1622366905212402, "rank": 2, "decoded_token": " detailed"}, "10575": {"logprob": -3.1622366905212402, "rank": 3, "decoded_token": " dog"}, "6468": {"logprob": -3.2247366905212402, "rank": 4, "decoded_token": " focus"}, "6165": {"logprob": -3.3497366905212402, "rank": 5, "decoded_token": " simple"}}, {"10575": {"logprob": -0.2919383645057678, "rank": 1, "decoded_token": " dog"}, "119075": {"logprob": -3.104438304901123, "rank": 2, "decoded_token": " Labrador"}, "116572": {"logprob": -3.479438304901123, "rank": 3, "decoded_token": " puppy"}, "15812": {"logprob": -4.104438304901123, "rank": 4, "decoded_token": " Lab"}, "2001": {"logprob": -4.479438304901123, "rank": 5, "decoded_token": " po"}}, {"7283": {"logprob": -1.594571590423584, "rank": 1, "decoded_token": " looking"}, "1408": {"logprob": -2.219571590423584, "rank": 2, "decoded_token": " on"}, "18970": {"logprob": -2.407071590423584, "rank": 3, "decoded_token": " sitting"}, "1454": {"logprob": -2.657071590423584, "rank": 4, "decoded_token": " with"}, "11589": {"logprob": -3.282071590423584, "rank": 5, "decoded_token": " gaz"}}, {"2015": {"logprob": -2.1110990047454834, "rank": 1, "decoded_token": " up"}, "7655": {"logprob": -2.1735990047454834, "rank": 2, "decoded_token": " directly"}, "1935": {"logprob": -2.4860990047454834, "rank": 3, "decoded_token": " int"}, "4524": {"logprob": -3.0485990047454834, "rank": 4, "decoded_token": " cur"}, "40022": {"logprob": -3.2985990047454834, "rank": 5, "decoded_token": " upward"}}, {"1454": {"logprob": -1.676759123802185, "rank": 1, "decoded_token": " with"}, "1513": {"logprob": -2.1142592430114746, "rank": 2, "decoded_token": " at"}, "4914": {"logprob": -2.5517592430114746, "rank": 3, "decoded_token": " thought"}, "1935": {"logprob": -2.8642592430114746, "rank": 4, "decoded_token": " int"}, "1408": {"logprob": -2.8642592430114746, "rank": 5, "decoded_token": " on"}}, {"1261": {"logprob": -1.243793249130249, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -1.931293249130249, "rank": 2, "decoded_token": " an"}, "2246": {"logprob": -2.618793249130249, "rank": 3, "decoded_token": " its"}, "9924": {"logprob": -2.618793249130249, "rank": 4, "decoded_token": " wide"}, "14781": {"logprob": -3.743793249130249, "rank": 5, "decoded_token": " focused"}}, {"26517": {"logprob": -1.6467483043670654, "rank": 1, "decoded_token": " calm"}, "14781": {"logprob": -2.5842483043670654, "rank": 2, "decoded_token": " focused"}, "29691": {"logprob": -2.7092483043670654, "rank": 3, "decoded_token": " contempl"}, "85596": {"logprob": -3.1467483043670654, "rank": 4, "decoded_token": " solemn"}, "16318": {"logprob": -3.2092483043670654, "rank": 5, "decoded_token": " neutral"}}, {"4818": {"logprob": -0.6443419456481934, "rank": 1, "decoded_token": " expression"}, "1311": {"logprob": -2.2068419456481934, "rank": 2, "decoded_token": " de"}, "1321": {"logprob": -2.5193419456481934, "rank": 3, "decoded_token": " and"}, "1044": {"logprob": -2.9568419456481934, "rank": 4, "decoded_token": ","}, "22131": {"logprob": -3.4568419456481934, "rank": 5, "decoded_token": " gaze"}}, {"1408": {"logprob": -0.7509063482284546, "rank": 1, "decoded_token": " on"}, "1338": {"logprob": -2.188406467437744, "rank": 2, "decoded_token": ".\n\n"}, "2136": {"logprob": -2.875906467437744, "rank": 3, "decoded_token": " over"}, "3675": {"logprob": -3.188406467437744, "rank": 4, "decoded_token": " against"}, "1046": {"logprob": -3.313406467437744, "rank": 5, "decoded_token": "."}}, {"1261": {"logprob": -0.41352635622024536, "rank": 1, "decoded_token": " a"}, "44130": {"logprob": -2.2885262966156006, "rank": 2, "decoded_token": " rust"}, "32656": {"logprob": -2.8510262966156006, "rank": 3, "decoded_token": " wooden"}, "2549": {"logprob": -3.6010262966156006, "rank": 4, "decoded_token": " what"}, "3403": {"logprob": -4.03852653503418, "rank": 5, "decoded_token": " text"}}, {"44130": {"logprob": -1.1644353866577148, "rank": 1, "decoded_token": " rust"}, "3403": {"logprob": -1.7269353866577148, "rank": 2, "decoded_token": " text"}, "32656": {"logprob": -1.7894353866577148, "rank": 3, "decoded_token": " wooden"}, "6165": {"logprob": -3.351935386657715, "rank": 4, "decoded_token": " simple"}, "1615": {"logprob": -3.601935386657715, "rank": 5, "decoded_token": " pl"}}, {"1290": {"logprob": -0.007089222315698862, "rank": 1, "decoded_token": "ic"}, "1121": {"logprob": -6.319589138031006, "rank": 2, "decoded_token": "y"}, "2981": {"logprob": -6.507089138031006, "rank": 3, "decoded_token": "ically"}, "12500": {"logprob": -6.694589138031006, "rank": 4, "decoded_token": "icated"}, "86794": {"logprob": -7.632089138031006, "rank": 5, "decoded_token": "-colored"}}, {"32656": {"logprob": -0.5749364495277405, "rank": 1, "decoded_token": " wooden"}, "1615": {"logprob": -2.4499363899230957, "rank": 2, "decoded_token": " pl"}, "1044": {"logprob": -2.6999363899230957, "rank": 3, "decoded_token": ","}, "12603": {"logprob": -2.8249363899230957, "rank": 4, "decoded_token": " wood"}, "4691": {"logprob": -3.3249363899230957, "rank": 5, "decoded_token": " surface"}}, {"1615": {"logprob": -0.6836710572242737, "rank": 1, "decoded_token": " pl"}, "7042": {"logprob": -2.433670997619629, "rank": 2, "decoded_token": " background"}, "4691": {"logprob": -2.433670997619629, "rank": 3, "decoded_token": " surface"}, "92504": {"logprob": -2.558670997619629, "rank": 4, "decoded_token": " backdrop"}, "9710": {"logprob": -3.058670997619629, "rank": 5, "decoded_token": " board"}}, {"2395": {"logprob": -0.063260018825531, "rank": 1, "decoded_token": "ank"}, "5933": {"logprob": -3.063260078430176, "rank": 2, "decoded_token": "anks"}, "122370": {"logprob": -4.313260078430176, "rank": 3, "decoded_token": "anking"}, "4713": {"logprob": -7.688260078430176, "rank": 4, "decoded_token": "atter"}, "58739": {"logprob": -8.563260078430176, "rank": 5, "decoded_token": "ANK"}}, {"7042": {"logprob": -1.4006984233856201, "rank": 1, "decoded_token": " background"}, "92504": {"logprob": -1.6506984233856201, "rank": 2, "decoded_token": " backdrop"}, "4691": {"logprob": -2.08819842338562, "rank": 3, "decoded_token": " surface"}, "1338": {"logprob": -2.83819842338562, "rank": 4, "decoded_token": ".\n\n"}, "9436": {"logprob": -2.96319842338562, "rank": 5, "decoded_token": " setting"}}, {"1338": {"logprob": -0.7081566452980042, "rank": 1, "decoded_token": ".\n\n"}, "1046": {"logprob": -0.8956566452980042, "rank": 2, "decoded_token": "."}, "1319": {"logprob": -4.145656585693359, "rank": 3, "decoded_token": " ("}, "2100": {"logprob": -4.395656585693359, "rank": 4, "decoded_token": ":\n\n"}, "1626": {"logprob": -4.708156585693359, "rank": 5, "decoded_token": ".\n"}}, {"2892": {"logprob": -2.5741822719573975, "rank": 1, "decoded_token": "To"}, "4393": {"logprob": -2.8866822719573975, "rank": 2, "decoded_token": "For"}, "11745": {"logprob": -3.1366822719573975, "rank": 3, "decoded_token": "Here"}, "12598": {"logprob": -3.1991822719573975, "rank": 4, "decoded_token": "Let"}, "1040": {"logprob": -3.2616822719573975, "rank": 5, "decoded_token": "("}}, {"38695": {"logprob": -1.3182454109191895, "rank": 1, "decoded_token": " clarify"}, "10035": {"logprob": -2.3807454109191895, "rank": 2, "decoded_token": " avoid"}, "11811": {"logprob": -3.0057454109191895, "rank": 3, "decoded_token": " ensure"}, "66370": {"logprob": -3.0682454109191895, "rank": 4, "decoded_token": " summarize"}, "36993": {"logprob": -3.4432454109191895, "rank": 5, "decoded_token": " strictly"}}, {"1044": {"logprob": -1.8413705825805664, "rank": 1, "decoded_token": ","}, "1278": {"logprob": -2.5913705825805664, "rank": 2, "decoded_token": " the"}, "1321": {"logprob": -2.9038705825805664, "rank": 3, "decoded_token": " and"}, "1877": {"logprob": -3.2163705825805664, "rank": 4, "decoded_token": ":\n"}, "4057": {"logprob": -3.5288705825805664, "rank": 5, "decoded_token": " based"}}, {"3226": {"logprob": -1.5737794637680054, "rank": 1, "decoded_token": " here"}, "1362": {"logprob": -2.698779582977295, "rank": 2, "decoded_token": " I"}, "1278": {"logprob": -2.761279582977295, "rank": 3, "decoded_token": " the"}, "2342": {"logprob": -2.886279582977295, "rank": 4, "decoded_token": " only"}, "2878": {"logprob": -2.948779582977295, "rank": 5, "decoded_token": " let"}}, {"2190": {"logprob": -1.2323426008224487, "rank": 1, "decoded_token": "\u2019s"}, "1681": {"logprob": -1.5448426008224487, "rank": 2, "decoded_token": "'s"}, "1584": {"logprob": -1.5448426008224487, "rank": 3, "decoded_token": " are"}, "1395": {"logprob": -2.0448427200317383, "rank": 4, "decoded_token": " is"}, "6510": {"logprob": -4.607342720031738, "rank": 5, "decoded_token": "with"}}, {"1278": {"logprob": -1.6423555612564087, "rank": 1, "decoded_token": " the"}, "1261": {"logprob": -1.7048555612564087, "rank": 2, "decoded_token": " a"}, "2606": {"logprob": -3.267355442047119, "rank": 3, "decoded_token": " how"}, "1420": {"logprob": -3.329855442047119, "rank": 4, "decoded_token": " an"}, "2342": {"logprob": -3.517355442047119, "rank": 5, "decoded_token": " only"}}, {"6298": {"logprob": -2.6795427799224854, "rank": 1, "decoded_token": " correct"}, "4811": {"logprob": -3.3670427799224854, "rank": 2, "decoded_token": " specific"}, "17793": {"logprob": -3.6795427799224854, "rank": 3, "decoded_token": " summary"}, "15115": {"logprob": -3.8045427799224854, "rank": 4, "decoded_token": " detailed"}, "6468": {"logprob": -3.8670427799224854, "rank": 5, "decoded_token": " focus"}}, {"6360": {"logprob": -2.9291810989379883, "rank": 1, "decoded_token": " description"}, "6468": {"logprob": -3.4291810989379883, "rank": 2, "decoded_token": " focus"}, "7980": {"logprob": -3.5541810989379883, "rank": 3, "decoded_token": " sequence"}, "1321": {"logprob": -3.5541810989379883, "rank": 4, "decoded_token": " and"}, "44433": {"logprob": -3.8041810989379883, "rank": 5, "decoded_token": " breakdown"}}, {"1394": {"logprob": -1.4118860960006714, "rank": 1, "decoded_token": " for"}, "1307": {"logprob": -2.161886215209961, "rank": 2, "decoded_token": " of"}, "4057": {"logprob": -2.599386215209961, "rank": 3, "decoded_token": " based"}, "30557": {"logprob": -3.724386215209961, "rank": 4, "decoded_token": " focusing"}, "2342": {"logprob": -3.911886215209961, "rank": 5, "decoded_token": " only"}}, {"1278": {"logprob": -1.170920729637146, "rank": 1, "decoded_token": " the"}, "2744": {"logprob": -2.4834208488464355, "rank": 2, "decoded_token": " each"}, "2143": {"logprob": -3.1709208488464355, "rank": 3, "decoded_token": " your"}, "1747": {"logprob": -3.2959208488464355, "rank": 4, "decoded_token": " all"}, "2342": {"logprob": -3.3584208488464355, "rank": 5, "decoded_token": " only"}}, {"5662": {"logprob": -2.8693411350250244, "rank": 1, "decoded_token": " provided"}, "5719": {"logprob": -3.1193411350250244, "rank": 2, "decoded_token": " initial"}, "8061": {"logprob": -3.3693411350250244, "rank": 3, "decoded_token": " images"}, "12432": {"logprob": -3.4318411350250244, "rank": 4, "decoded_token": " fourth"}, "2667": {"logprob": -3.4943411350250244, "rank": 5, "decoded_token": " second"}}, {"7244": {"logprob": -2.477940559387207, "rank": 1, "decoded_token": " black"}, "10575": {"logprob": -2.727940559387207, "rank": 2, "decoded_token": " dog"}, "3937": {"logprob": -2.727940559387207, "rank": 3, "decoded_token": " image"}, "8061": {"logprob": -2.790440559387207, "rank": 4, "decoded_token": " images"}, "2016": {"logprob": -2.852940559387207, "rank": 5, "decoded_token": " set"}}, {"10575": {"logprob": -0.09506329894065857, "rank": 1, "decoded_token": " dog"}, "3937": {"logprob": -3.8450632095336914, "rank": 2, "decoded_token": " image"}, "63524": {"logprob": -4.470063209533691, "rank": 3, "decoded_token": "dog"}, "3028": {"logprob": -4.970063209533691, "rank": 4, "decoded_token": "-d"}, "1321": {"logprob": -5.220063209533691, "rank": 5, "decoded_token": " and"}}, {"3937": {"logprob": -0.6087309122085571, "rank": 1, "decoded_token": " image"}, "2100": {"logprob": -2.7337307929992676, "rank": 2, "decoded_token": ":\n\n"}, "1294": {"logprob": -3.2337307929992676, "rank": 3, "decoded_token": " in"}, "13083": {"logprob": -3.3587307929992676, "rank": 4, "decoded_token": " picture"}, "1877": {"logprob": -3.3587307929992676, "rank": 5, "decoded_token": ":\n"}}, {"1877": {"logprob": -1.969537615776062, "rank": 1, "decoded_token": ":\n"}, "2100": {"logprob": -2.0945377349853516, "rank": 2, "decoded_token": ":\n\n"}, "2342": {"logprob": -2.4695377349853516, "rank": 3, "decoded_token": " only"}, "13703": {"logprob": -3.0945377349853516, "rank": 4, "decoded_token": " specifically"}, "9412": {"logprob": -3.1570377349853516, "rank": 5, "decoded_token": " alone"}}, {"1065": {"logprob": -1.341880202293396, "rank": 1, "decoded_token": "A"}, "1784": {"logprob": -1.841880202293396, "rank": 2, "decoded_token": "The"}, "1256": {"logprob": -1.966880202293396, "rank": 3, "decoded_token": " "}, "1438": {"logprob": -2.4668803215026855, "rank": 4, "decoded_token": "**"}, "1045": {"logprob": -2.5293803215026855, "rank": 5, "decoded_token": "-"}}, {"7244": {"logprob": -0.6517431139945984, "rank": 1, "decoded_token": " black"}, "6231": {"logprob": -2.714243173599243, "rank": 2, "decoded_token": " close"}, "85596": {"logprob": -3.026743173599243, "rank": 3, "decoded_token": " solemn"}, "14781": {"logprob": -3.401743173599243, "rank": 4, "decoded_token": " focused"}, "26517": {"logprob": -3.526743173599243, "rank": 5, "decoded_token": " calm"}}, {"10575": {"logprob": -0.10892026871442795, "rank": 1, "decoded_token": " dog"}, "119075": {"logprob": -3.7964203357696533, "rank": 2, "decoded_token": " Labrador"}, "116572": {"logprob": -3.9839203357696533, "rank": 3, "decoded_token": " puppy"}, "1044": {"logprob": -4.983920097351074, "rank": 4, "decoded_token": ","}, "94057": {"logprob": -5.171420097351074, "rank": 5, "decoded_token": " canine"}}, {"11589": {"logprob": -1.534816861152649, "rank": 1, "decoded_token": " gaz"}, "1395": {"logprob": -1.722316861152649, "rank": 2, "decoded_token": " is"}, "1454": {"logprob": -2.0348167419433594, "rank": 3, "decoded_token": " with"}, "53048": {"logprob": -2.2848167419433594, "rank": 4, "decoded_token": " sits"}, "10637": {"logprob": -2.6598167419433594, "rank": 5, "decoded_token": " looks"}}, {"1264": {"logprob": -0.4754399061203003, "rank": 1, "decoded_token": "es"}, "1302": {"logprob": -0.9754399061203003, "rank": 2, "decoded_token": "ing"}, "1944": {"logprob": -8.16294002532959, "rank": 3, "decoded_token": "ely"}, "47885": {"logprob": -8.16294002532959, "rank": 4, "decoded_token": "edly"}, "15006": {"logprob": -8.41294002532959, "rank": 5, "decoded_token": "ingly"}}, {"1935": {"logprob": -1.7025537490844727, "rank": 1, "decoded_token": " int"}, "7655": {"logprob": -2.0775537490844727, "rank": 2, "decoded_token": " directly"}, "40022": {"logprob": -2.2650537490844727, "rank": 3, "decoded_token": " upward"}, "2015": {"logprob": -2.4525537490844727, "rank": 4, "decoded_token": " up"}, "74606": {"logprob": -2.7025537490844727, "rank": 5, "decoded_token": " upwards"}}, {"3929": {"logprob": -0.0234219990670681, "rank": 1, "decoded_token": "ently"}, "2749": {"logprob": -5.335921764373779, "rank": 2, "decoded_token": "ros"}, "1533": {"logprob": -6.398421764373779, "rank": 3, "decoded_token": "ang"}, "3923": {"logprob": -6.523421764373779, "rank": 4, "decoded_token": "ively"}, "20626": {"logprob": -6.773421764373779, "rank": 5, "decoded_token": "rep"}}, {"40022": {"logprob": -0.8856239318847656, "rank": 1, "decoded_token": " upward"}, "74606": {"logprob": -1.5106239318847656, "rank": 2, "decoded_token": " upwards"}, "1454": {"logprob": -2.2606239318847656, "rank": 3, "decoded_token": " with"}, "8848": {"logprob": -2.7606239318847656, "rank": 4, "decoded_token": " forward"}, "1513": {"logprob": -3.5106239318847656, "rank": 5, "decoded_token": " at"}}, {"1454": {"logprob": -1.170392632484436, "rank": 1, "decoded_token": " with"}, "1408": {"logprob": -1.295392632484436, "rank": 2, "decoded_token": " on"}, "1562": {"logprob": -2.1703925132751465, "rank": 3, "decoded_token": " from"}, "3016": {"logprob": -2.4828925132751465, "rank": 4, "decoded_token": " while"}, "3675": {"logprob": -3.1703925132751465, "rank": 5, "decoded_token": " against"}}, {"1261": {"logprob": -0.8887192010879517, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -1.7637192010879517, "rank": 2, "decoded_token": " an"}, "2246": {"logprob": -2.138719081878662, "rank": 3, "decoded_token": " its"}, "9924": {"logprob": -2.888719081878662, "rank": 4, "decoded_token": " wide"}, "26517": {"logprob": -3.888719081878662, "rank": 5, "decoded_token": " calm"}}, {"26517": {"logprob": -1.2183846235275269, "rank": 1, "decoded_token": " calm"}, "2169": {"logprob": -2.8433847427368164, "rank": 2, "decoded_token": " ser"}, "16318": {"logprob": -3.0933847427368164, "rank": 3, "decoded_token": " neutral"}, "14781": {"logprob": -3.2183847427368164, "rank": 4, "decoded_token": " focused"}, "6444": {"logprob": -3.2183847427368164, "rank": 5, "decoded_token": " soft"}}, {"4818": {"logprob": -0.4964141249656677, "rank": 1, "decoded_token": " expression"}, "1321": {"logprob": -1.9339141845703125, "rank": 2, "decoded_token": " and"}, "1311": {"logprob": -2.4339141845703125, "rank": 3, "decoded_token": " de"}, "1044": {"logprob": -3.3089141845703125, "rank": 4, "decoded_token": ","}, "2985": {"logprob": -3.4964141845703125, "rank": 5, "decoded_token": " look"}}, {"1408": {"logprob": -0.8785426616668701, "rank": 1, "decoded_token": " on"}, "1562": {"logprob": -2.06604266166687, "rank": 2, "decoded_token": " from"}, "18970": {"logprob": -2.31604266166687, "rank": 3, "decoded_token": " sitting"}, "3675": {"logprob": -2.62854266166687, "rank": 4, "decoded_token": " against"}, "38235": {"logprob": -3.06604266166687, "rank": 5, "decoded_token": " resting"}}, {"1261": {"logprob": -0.3605937659740448, "rank": 1, "decoded_token": " a"}, "32656": {"logprob": -2.923093795776367, "rank": 2, "decoded_token": " wooden"}, "17253": {"logprob": -3.110593795776367, "rank": 3, "decoded_token": " weather"}, "3403": {"logprob": -3.173093795776367, "rank": 4, "decoded_token": " text"}, "44130": {"logprob": -3.298093795776367, "rank": 5, "decoded_token": " rust"}}, {"44130": {"logprob": -1.4187138080596924, "rank": 1, "decoded_token": " rust"}, "32656": {"logprob": -1.4812138080596924, "rank": 2, "decoded_token": " wooden"}, "3403": {"logprob": -1.6687138080596924, "rank": 3, "decoded_token": " text"}, "17253": {"logprob": -2.4812138080596924, "rank": 4, "decoded_token": " weather"}, "8500": {"logprob": -4.168713569641113, "rank": 5, "decoded_token": " dark"}}, {"1290": {"logprob": -0.28350138664245605, "rank": 1, "decoded_token": "ic"}, "1970": {"logprob": -2.283501386642456, "rank": 2, "decoded_token": "led"}, "1121": {"logprob": -2.971001386642456, "rank": 3, "decoded_token": "y"}, "2981": {"logprob": -4.283501625061035, "rank": 4, "decoded_token": "ically"}, "11395": {"logprob": -4.471001625061035, "rank": 5, "decoded_token": "iced"}}, {"32656": {"logprob": -0.7727869749069214, "rank": 1, "decoded_token": " wooden"}, "1044": {"logprob": -1.4602869749069214, "rank": 2, "decoded_token": ","}, "3403": {"logprob": -2.585287094116211, "rank": 3, "decoded_token": " text"}, "1615": {"logprob": -2.835287094116211, "rank": 4, "decoded_token": " pl"}, "12603": {"logprob": -3.335287094116211, "rank": 5, "decoded_token": " wood"}}, {"1615": {"logprob": -0.7352191805839539, "rank": 1, "decoded_token": " pl"}, "4691": {"logprob": -2.1102192401885986, "rank": 2, "decoded_token": " surface"}, "9710": {"logprob": -2.6102192401885986, "rank": 3, "decoded_token": " board"}, "7042": {"logprob": -2.9852192401885986, "rank": 4, "decoded_token": " background"}, "92504": {"logprob": -3.1102192401885986, "rank": 5, "decoded_token": " backdrop"}}, {"2395": {"logprob": -0.13373544812202454, "rank": 1, "decoded_token": "ank"}, "5933": {"logprob": -2.133735418319702, "rank": 2, "decoded_token": "anks"}, "122370": {"logprob": -6.883735656738281, "rank": 3, "decoded_token": "anking"}, "11847": {"logprob": -8.071235656738281, "rank": 4, "decoded_token": "anned"}, "2077": {"logprob": -8.071235656738281, "rank": 5, "decoded_token": "ink"}}, {"7042": {"logprob": -1.197163701057434, "rank": 1, "decoded_token": " background"}, "4691": {"logprob": -1.572163701057434, "rank": 2, "decoded_token": " surface"}, "92504": {"logprob": -1.884663701057434, "rank": 3, "decoded_token": " backdrop"}, "26228": {"logprob": -3.3221635818481445, "rank": 4, "decoded_token": " texture"}, "9436": {"logprob": -3.3846635818481445, "rank": 5, "decoded_token": " setting"}}, {"1046": {"logprob": -0.13945358991622925, "rank": 1, "decoded_token": "."}, "1338": {"logprob": -2.576953649520874, "rank": 2, "decoded_token": ".\n\n"}, "1294": {"logprob": -4.764453411102295, "rank": 3, "decoded_token": " in"}, "1044": {"logprob": -5.701953411102295, "rank": 4, "decoded_token": ","}, "1319": {"logprob": -5.889453411102295, "rank": 5, "decoded_token": " ("}}, {"2": {"logprob": -0.5447223782539368, "rank": 1, "decoded_token": ""}, "1319": {"logprob": -2.607222318649292, "rank": 2, "decoded_token": " ("}, "9380": {"logprob": -3.669722318649292, "rank": 3, "decoded_token": " Here"}, "1531": {"logprob": -3.919722318649292, "rank": 4, "decoded_token": " The"}, "2898": {"logprob": -4.107222557067871, "rank": 5, "decoded_token": " For"}}]]]
\ No newline at end of file
diff --git a/asian_ai/performance/vLLM/tests/models/fixtures/mistral_small_3_chat.json b/asian_ai/performance/vLLM/tests/models/fixtures/mistral_small_3_chat.json
new file mode 100644
index 0000000000000000000000000000000000000000..9d65cd0bd6dd836a1d2e810e9bbd46a547fa09eb
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/fixtures/mistral_small_3_chat.json
@@ -0,0 +1 @@
+[[[1784, 3937, 6122, 1261, 7244, 10575, 28528, 1408, 1261, 32656, 11237, 1044, 7283, 2015, 1454, 1261, 38462, 4818, 1046, 2], "The image shows a black dog lying on a wooden floor, looking up with a curious expression.", [{"1784": {"logprob": -0.4740446209907532, "rank": 1, "decoded_token": "The"}, "1065": {"logprob": -1.0990445613861084, "rank": 2, "decoded_token": "A"}, "4380": {"logprob": -3.3490445613861084, "rank": 3, "decoded_token": "This"}, "1785": {"logprob": -5.0990447998046875, "rank": 4, "decoded_token": "In"}, "11745": {"logprob": -6.4740447998046875, "rank": 5, "decoded_token": "Here"}}, {"3937": {"logprob": -0.06349722295999527, "rank": 1, "decoded_token": " image"}, "7244": {"logprob": -2.813497304916382, "rank": 2, "decoded_token": " black"}, "16649": {"logprob": -7.563497066497803, "rank": 3, "decoded_token": " photo"}, "18390": {"logprob": -7.688497066497803, "rank": 4, "decoded_token": " photograph"}, "10575": {"logprob": -8.438497543334961, "rank": 5, "decoded_token": " dog"}}, {"6122": {"logprob": -0.25453490018844604, "rank": 1, "decoded_token": " shows"}, "6971": {"logprob": -1.8795349597930908, "rank": 2, "decoded_token": " features"}, "51948": {"logprob": -2.754534959793091, "rank": 3, "decoded_token": " depicts"}, "25981": {"logprob": -5.629534721374512, "rank": 4, "decoded_token": " displays"}, "1395": {"logprob": -6.129534721374512, "rank": 5, "decoded_token": " is"}}, {"1261": {"logprob": -0.0001245659514097497, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -9.00012493133545, "rank": 2, "decoded_token": " an"}, "1278": {"logprob": -14.25012493133545, "rank": 3, "decoded_token": " the"}, "7244": {"logprob": -14.87512493133545, "rank": 4, "decoded_token": " black"}, "1925": {"logprob": -16.125123977661133, "rank": 5, "decoded_token": " one"}}, {"7244": {"logprob": -0.009403933770954609, "rank": 1, "decoded_token": " black"}, "6231": {"logprob": -5.259403705596924, "rank": 2, "decoded_token": " close"}, "16450": {"logprob": -6.759403705596924, "rank": 3, "decoded_token": " sle"}, "8500": {"logprob": -7.009403705596924, "rank": 4, "decoded_token": " dark"}, "4329": {"logprob": -7.696903705596924, "rank": 5, "decoded_token": " large"}}, {"10575": {"logprob": -0.7522680163383484, "rank": 1, "decoded_token": " dog"}, "119075": {"logprob": -1.0022680759429932, "rank": 2, "decoded_token": " Labrador"}, "116572": {"logprob": -1.8772680759429932, "rank": 3, "decoded_token": " puppy"}, "8636": {"logprob": -5.627267837524414, "rank": 4, "decoded_token": " lab"}, "15812": {"logprob": -5.814767837524414, "rank": 5, "decoded_token": " Lab"}}, {"28528": {"logprob": -0.2941223084926605, "rank": 1, "decoded_token": " lying"}, "7283": {"logprob": -2.1691222190856934, "rank": 2, "decoded_token": " looking"}, "1454": {"logprob": -2.5441222190856934, "rank": 3, "decoded_token": " with"}, "60700": {"logprob": -3.2941222190856934, "rank": 4, "decoded_token": " laying"}, "18970": {"logprob": -4.794122219085693, "rank": 5, "decoded_token": " sitting"}}, {"1408": {"logprob": -0.3170951306819916, "rank": 1, "decoded_token": " on"}, "3151": {"logprob": -1.317095160484314, "rank": 2, "decoded_token": " down"}, "14038": {"logprob": -7.3170952796936035, "rank": 3, "decoded_token": " flat"}, "104248": {"logprob": -7.4420952796936035, "rank": 4, "decoded_token": " comfortably"}, "1321": {"logprob": -7.6920952796936035, "rank": 5, "decoded_token": " and"}}, {"1261": {"logprob": -0.08228635042905807, "rank": 1, "decoded_token": " a"}, "2246": {"logprob": -3.2072863578796387, "rank": 2, "decoded_token": " its"}, "32656": {"logprob": -3.3322863578796387, "rank": 3, "decoded_token": " wooden"}, "3977": {"logprob": -6.957286357879639, "rank": 4, "decoded_token": " top"}, "1278": {"logprob": -7.207286357879639, "rank": 5, "decoded_token": " the"}}, {"32656": {"logprob": -0.03605202957987785, "rank": 1, "decoded_token": " wooden"}, "3403": {"logprob": -3.9110519886016846, "rank": 2, "decoded_token": " text"}, "44130": {"logprob": -4.911052227020264, "rank": 3, "decoded_token": " rust"}, "12603": {"logprob": -6.036052227020264, "rank": 4, "decoded_token": " wood"}, "8500": {"logprob": -6.473552227020264, "rank": 5, "decoded_token": " dark"}}, {"11237": {"logprob": -0.6433407068252563, "rank": 1, "decoded_token": " floor"}, "4691": {"logprob": -0.7683407068252563, "rank": 2, "decoded_token": " surface"}, "1615": {"logprob": -5.268340587615967, "rank": 3, "decoded_token": " pl"}, "3403": {"logprob": -6.018340587615967, "rank": 4, "decoded_token": " text"}, "18645": {"logprob": -7.143340587615967, "rank": 5, "decoded_token": " flo"}}, {"1044": {"logprob": -0.6826052665710449, "rank": 1, "decoded_token": ","}, "1321": {"logprob": -1.682605266571045, "rank": 2, "decoded_token": " and"}, "7283": {"logprob": -1.807605266571045, "rank": 3, "decoded_token": " looking"}, "1046": {"logprob": -2.682605266571045, "rank": 4, "decoded_token": "."}, "1454": {"logprob": -3.182605266571045, "rank": 5, "decoded_token": " with"}}, {"7283": {"logprob": -0.07239976525306702, "rank": 1, "decoded_token": " looking"}, "11589": {"logprob": -3.197399854660034, "rank": 2, "decoded_token": " gaz"}, "35542": {"logprob": -3.822399854660034, "rank": 3, "decoded_token": " staring"}, "1454": {"logprob": -6.384899616241455, "rank": 4, "decoded_token": " with"}, "22116": {"logprob": -6.572399616241455, "rank": 5, "decoded_token": " facing"}}, {"2015": {"logprob": -0.9646494388580322, "rank": 2, "decoded_token": " up"}, "7655": {"logprob": -0.9646494388580322, "rank": 1, "decoded_token": " directly"}, "74606": {"logprob": -2.0896494388580322, "rank": 3, "decoded_token": " upwards"}, "40022": {"logprob": -3.0896494388580322, "rank": 4, "decoded_token": " upward"}, "1935": {"logprob": -4.152149200439453, "rank": 5, "decoded_token": " int"}}, {"1454": {"logprob": -0.8447978496551514, "rank": 1, "decoded_token": " with"}, "1513": {"logprob": -1.2197978496551514, "rank": 2, "decoded_token": " at"}, "41132": {"logprob": -2.2197978496551514, "rank": 3, "decoded_token": " attent"}, "1935": {"logprob": -2.9697978496551514, "rank": 4, "decoded_token": " int"}, "7655": {"logprob": -3.0947978496551514, "rank": 5, "decoded_token": " directly"}}, {"1261": {"logprob": -0.7162021994590759, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -1.3412022590637207, "rank": 2, "decoded_token": " an"}, "41132": {"logprob": -2.2162022590637207, "rank": 3, "decoded_token": " attent"}, "2246": {"logprob": -3.2162022590637207, "rank": 4, "decoded_token": " its"}, "38462": {"logprob": -3.9662022590637207, "rank": 5, "decoded_token": " curious"}}, {"38462": {"logprob": -0.7836517095565796, "rank": 1, "decoded_token": " curious"}, "26517": {"logprob": -1.8461517095565796, "rank": 2, "decoded_token": " calm"}, "26905": {"logprob": -2.533651828765869, "rank": 3, "decoded_token": " gentle"}, "11304": {"logprob": -3.408651828765869, "rank": 4, "decoded_token": " serious"}, "97680": {"logprob": -3.596151828765869, "rank": 5, "decoded_token": " thoughtful"}}, {"4818": {"logprob": -0.047154705971479416, "rank": 1, "decoded_token": " expression"}, "1321": {"logprob": -3.922154664993286, "rank": 2, "decoded_token": " and"}, "1505": {"logprob": -4.047154903411865, "rank": 3, "decoded_token": " or"}, "22131": {"logprob": -4.797154903411865, "rank": 4, "decoded_token": " gaze"}, "1044": {"logprob": -9.047154426574707, "rank": 5, "decoded_token": ","}}, {"1046": {"logprob": -0.0008031480247154832, "rank": 1, "decoded_token": "."}, "1408": {"logprob": -7.250802993774414, "rank": 2, "decoded_token": " on"}, "1321": {"logprob": -10.500802993774414, "rank": 3, "decoded_token": " and"}, "1338": {"logprob": -11.000802993774414, "rank": 4, "decoded_token": ".\n\n"}, "3016": {"logprob": -11.500802993774414, "rank": 5, "decoded_token": " while"}}, {"2": {"logprob": -0.0008517451351508498, "rank": 1, "decoded_token": " "}, "1032": {"logprob": -7.125851631164551, "rank": 2, "decoded_token": " "}, "1256": {"logprob": -10.00085163116455, "rank": 3, "decoded_token": " The"}}]], [[1049, 1046, 1349, 7244, 10575, 1395, 28528, 1408, 1261, 32656, 11237, 1044, 7283, 2015, 1513, 1278, 13424, 1626, 1050, 1046, 1349, 10726, 1290, 3719, 1307, 122203, 35463, 1454, 11223, 1321, 95746, 24765, 2425, 1261, 6133, 21283, 1046, 2], "1. A black dog is lying on a wooden floor, looking up at the camera.\n2. A scenic view of rugged mountains with green and rocky terrain under a clear sky.", [{"1049": {"logprob": -0.05050129443407059, "rank": 1, "decoded_token": "1"}, "11745": {"logprob": -3.5505013465881348, "rank": 2, "decoded_token": "Here"}, "69957": {"logprob": -4.175501346588135, "rank": 3, "decoded_token": "Sure"}, "117991": {"logprob": -6.175501346588135, "rank": 4, "decoded_token": "Certain"}, "1045": {"logprob": -6.550501346588135, "rank": 5, "decoded_token": "-"}}, {"1046": {"logprob": -5.364403477869928e-06, "rank": 1, "decoded_token": "."}, "1041": {"logprob": -12.500005722045898, "rank": 2, "decoded_token": ")"}, "1058": {"logprob": -13.875005722045898, "rank": 3, "decoded_token": ":"}, "1044": {"logprob": -15.687505722045898, "rank": 4, "decoded_token": ","}, "1045": {"logprob": -15.875005722045898, "rank": 5, "decoded_token": "-"}}, {"1349": {"logprob": -0.4890742003917694, "rank": 1, "decoded_token": " A"}, "1531": {"logprob": -1.1140742301940918, "rank": 2, "decoded_token": " The"}, "1603": {"logprob": -3.364074230194092, "rank": 3, "decoded_token": " **"}, "1656": {"logprob": -4.364074230194092, "rank": 4, "decoded_token": " In"}, "2409": {"logprob": -4.989074230194092, "rank": 5, "decoded_token": " This"}}, {"7244": {"logprob": -0.08685152232646942, "rank": 1, "decoded_token": " black"}, "6231": {"logprob": -3.4618515968322754, "rank": 2, "decoded_token": " close"}, "16450": {"logprob": -3.5868515968322754, "rank": 3, "decoded_token": " sle"}, "4329": {"logprob": -4.899351596832275, "rank": 4, "decoded_token": " large"}, "8500": {"logprob": -5.399351596832275, "rank": 5, "decoded_token": " dark"}}, {"10575": {"logprob": -0.20338763296604156, "rank": 1, "decoded_token": " dog"}, "116572": {"logprob": -1.8283876180648804, "rank": 2, "decoded_token": " puppy"}, "119075": {"logprob": -3.95338773727417, "rank": 3, "decoded_token": " Labrador"}, "28404": {"logprob": -6.95338773727417, "rank": 4, "decoded_token": " pup"}, "8636": {"logprob": -7.07838773727417, "rank": 5, "decoded_token": " lab"}}, {"1395": {"logprob": -0.532414972782135, "rank": 1, "decoded_token": " is"}, "22524": {"logprob": -1.7824149131774902, "rank": 2, "decoded_token": " lies"}, "1454": {"logprob": -2.1574149131774902, "rank": 3, "decoded_token": " with"}, "10637": {"logprob": -3.2824149131774902, "rank": 4, "decoded_token": " looks"}, "28528": {"logprob": -3.4074149131774902, "rank": 5, "decoded_token": " lying"}}, {"28528": {"logprob": -0.4258010685443878, "rank": 1, "decoded_token": " lying"}, "7283": {"logprob": -1.6758010387420654, "rank": 2, "decoded_token": " looking"}, "60700": {"logprob": -2.9258010387420654, "rank": 3, "decoded_token": " laying"}, "38235": {"logprob": -3.6758010387420654, "rank": 4, "decoded_token": " resting"}, "18970": {"logprob": -3.6758010387420654, "rank": 5, "decoded_token": " sitting"}}, {"1408": {"logprob": -0.3588743805885315, "rank": 1, "decoded_token": " on"}, "3151": {"logprob": -1.2338743209838867, "rank": 2, "decoded_token": " down"}, "41132": {"logprob": -6.358874320983887, "rank": 3, "decoded_token": " attent"}, "14038": {"logprob": -6.546374320983887, "rank": 4, "decoded_token": " flat"}, "1321": {"logprob": -6.733874320983887, "rank": 5, "decoded_token": " and"}}, {"1261": {"logprob": -0.07801607996225357, "rank": 1, "decoded_token": " a"}, "2246": {"logprob": -2.9530160427093506, "rank": 2, "decoded_token": " its"}, "32656": {"logprob": -4.20301628112793, "rank": 3, "decoded_token": " wooden"}, "1278": {"logprob": -5.20301628112793, "rank": 4, "decoded_token": " the"}, "3977": {"logprob": -6.57801628112793, "rank": 5, "decoded_token": " top"}}, {"32656": {"logprob": -0.06541638821363449, "rank": 1, "decoded_token": " wooden"}, "3403": {"logprob": -3.4404163360595703, "rank": 2, "decoded_token": " text"}, "44130": {"logprob": -3.9404163360595703, "rank": 3, "decoded_token": " rust"}, "17253": {"logprob": -5.81541633605957, "rank": 4, "decoded_token": " weather"}, "12603": {"logprob": -5.94041633605957, "rank": 5, "decoded_token": " wood"}}, {"11237": {"logprob": -0.4574064016342163, "rank": 1, "decoded_token": " floor"}, "4691": {"logprob": -1.0824064016342163, "rank": 2, "decoded_token": " surface"}, "1615": {"logprob": -4.082406520843506, "rank": 3, "decoded_token": " pl"}, "3403": {"logprob": -5.207406520843506, "rank": 4, "decoded_token": " text"}, "28984": {"logprob": -6.582406520843506, "rank": 5, "decoded_token": " deck"}}, {"1044": {"logprob": -0.9594833850860596, "rank": 1, "decoded_token": ","}, "7283": {"logprob": -1.2094833850860596, "rank": 2, "decoded_token": " looking"}, "1321": {"logprob": -2.2094833850860596, "rank": 3, "decoded_token": " and"}, "1454": {"logprob": -2.4594833850860596, "rank": 4, "decoded_token": " with"}, "1626": {"logprob": -2.5844833850860596, "rank": 5, "decoded_token": ".\n"}}, {"7283": {"logprob": -0.15972694754600525, "rank": 1, "decoded_token": " looking"}, "11589": {"logprob": -2.534726858139038, "rank": 2, "decoded_token": " gaz"}, "35542": {"logprob": -2.909726858139038, "rank": 3, "decoded_token": " staring"}, "22116": {"logprob": -6.034727096557617, "rank": 4, "decoded_token": " facing"}, "1454": {"logprob": -6.409727096557617, "rank": 5, "decoded_token": " with"}}, {"2015": {"logprob": -0.894250750541687, "rank": 1, "decoded_token": " up"}, "7655": {"logprob": -1.269250750541687, "rank": 2, "decoded_token": " directly"}, "74606": {"logprob": -1.769250750541687, "rank": 3, "decoded_token": " upwards"}, "40022": {"logprob": -2.6442508697509766, "rank": 4, "decoded_token": " upward"}, "1935": {"logprob": -4.081750869750977, "rank": 5, "decoded_token": " int"}}, {"1513": {"logprob": -0.5085363388061523, "rank": 1, "decoded_token": " at"}, "1454": {"logprob": -1.5085363388061523, "rank": 2, "decoded_token": " with"}, "1626": {"logprob": -2.6335363388061523, "rank": 3, "decoded_token": ".\n"}, "1935": {"logprob": -3.3835363388061523, "rank": 4, "decoded_token": " int"}, "41132": {"logprob": -3.6335363388061523, "rank": 5, "decoded_token": " attent"}}, {"1278": {"logprob": -0.0010482537327334285, "rank": 1, "decoded_token": " the"}, "4433": {"logprob": -7.0010480880737305, "rank": 2, "decoded_token": " something"}, "2246": {"logprob": -10.25104808807373, "rank": 3, "decoded_token": " its"}, "1261": {"logprob": -10.25104808807373, "rank": 4, "decoded_token": " a"}, "1636": {"logprob": -10.50104808807373, "rank": 5, "decoded_token": " you"}}, {"13424": {"logprob": -0.0003800861886702478, "rank": 1, "decoded_token": " camera"}, "56268": {"logprob": -8.250380516052246, "rank": 2, "decoded_token": " viewer"}, "68439": {"logprob": -9.250380516052246, "rank": 3, "decoded_token": " photographer"}, "2965": {"logprob": -12.375380516052246, "rank": 4, "decoded_token": " person"}, "37967": {"logprob": -12.500380516052246, "rank": 5, "decoded_token": " ceiling"}}, {"1626": {"logprob": -0.34197133779525757, "rank": 1, "decoded_token": ".\n"}, "1454": {"logprob": -1.4669713973999023, "rank": 2, "decoded_token": " with"}, "1046": {"logprob": -3.3419713973999023, "rank": 3, "decoded_token": "."}, "1338": {"logprob": -3.9669713973999023, "rank": 4, "decoded_token": ".\n\n"}, "1935": {"logprob": -5.966971397399902, "rank": 5, "decoded_token": " int"}}, {"1050": {"logprob": -0.002148107625544071, "rank": 1, "decoded_token": "2"}, "1256": {"logprob": -6.877148151397705, "rank": 2, "decoded_token": " "}, "1293": {"logprob": -7.127148151397705, "rank": 3, "decoded_token": " "}, "1032": {"logprob": -8.252147674560547, "rank": 4, "decoded_token": " "}, "1049": {"logprob": -10.752147674560547, "rank": 5, "decoded_token": "1"}}, {"1046": {"logprob": -7.510157047363464e-06, "rank": 1, "decoded_token": "."}, "3590": {"logprob": -13.437507629394531, "rank": 2, "decoded_token": ".A"}, "1626": {"logprob": -13.437507629394531, "rank": 3, "decoded_token": ".\n"}, "48426": {"logprob": -13.687507629394531, "rank": 4, "decoded_token": ".The"}, "1044": {"logprob": -14.062507629394531, "rank": 5, "decoded_token": ","}}, {"1349": {"logprob": -0.2843300700187683, "rank": 1, "decoded_token": " A"}, "11826": {"logprob": -2.034330129623413, "rank": 2, "decoded_token": " Maj"}, "113465": {"logprob": -3.534330129623413, "rank": 3, "decoded_token": " Rug"}, "22468": {"logprob": -4.409329891204834, "rank": 4, "decoded_token": " Several"}, "1531": {"logprob": -4.534329891204834, "rank": 5, "decoded_token": " The"}}, {"10726": {"logprob": -1.3984904289245605, "rank": 1, "decoded_token": " scen"}, "122203": {"logprob": -1.7734904289245605, "rank": 2, "decoded_token": " rugged"}, "61082": {"logprob": -1.7734904289245605, "rank": 3, "decoded_token": " panor"}, "15375": {"logprob": -2.5234904289245605, "rank": 4, "decoded_token": " vast"}, "13770": {"logprob": -2.6484904289245605, "rank": 5, "decoded_token": " maj"}}, {"1290": {"logprob": -3.099436753473128e-06, "rank": 1, "decoded_token": "ic"}, "2981": {"logprob": -13.56250286102295, "rank": 2, "decoded_token": "ically"}, "1702": {"logprob": -14.31250286102295, "rank": 3, "decoded_token": "ice"}, "4965": {"logprob": -16.625003814697266, "rank": 4, "decoded_token": "etic"}, "4336": {"logprob": -16.687503814697266, "rank": 5, "decoded_token": "icro"}}, {"3719": {"logprob": -0.1252945065498352, "rank": 1, "decoded_token": " view"}, "28035": {"logprob": -2.8752944469451904, "rank": 2, "decoded_token": " landscape"}, "24361": {"logprob": -3.2502944469451904, "rank": 3, "decoded_token": " mountain"}, "127945": {"logprob": -5.1252946853637695, "rank": 4, "decoded_token": " mountainous"}, "1044": {"logprob": -5.3752946853637695, "rank": 5, "decoded_token": ","}}, {"1307": {"logprob": -0.09058280289173126, "rank": 1, "decoded_token": " of"}, "89995": {"logprob": -3.465582847595215, "rank": 2, "decoded_token": " showc"}, "6122": {"logprob": -3.715582847595215, "rank": 3, "decoded_token": " shows"}, "6971": {"logprob": -4.590582847595215, "rank": 4, "decoded_token": " features"}, "66583": {"logprob": -5.090582847595215, "rank": 5, "decoded_token": " captures"}}, {"122203": {"logprob": -0.5323622226715088, "rank": 1, "decoded_token": " rugged"}, "1261": {"logprob": -2.032362222671509, "rank": 2, "decoded_token": " a"}, "6245": {"logprob": -2.532362222671509, "rank": 3, "decoded_token": " multiple"}, "127945": {"logprob": -3.157362222671509, "rank": 4, "decoded_token": " mountainous"}, "35463": {"logprob": -3.532362222671509, "rank": 5, "decoded_token": " mountains"}}, {"35463": {"logprob": -0.6520033478736877, "rank": 1, "decoded_token": " mountains"}, "1044": {"logprob": -1.027003288269043, "rank": 2, "decoded_token": ","}, "24361": {"logprob": -2.527003288269043, "rank": 3, "decoded_token": " mountain"}, "127945": {"logprob": -3.902003288269043, "rank": 4, "decoded_token": " mountainous"}, "11223": {"logprob": -4.652003288269043, "rank": 5, "decoded_token": " green"}}, {"1454": {"logprob": -0.39697548747062683, "rank": 1, "decoded_token": " with"}, "13875": {"logprob": -2.146975517272949, "rank": 2, "decoded_token": " covered"}, "1321": {"logprob": -2.271975517272949, "rank": 3, "decoded_token": " and"}, "2425": {"logprob": -3.459475517272949, "rank": 4, "decoded_token": " under"}, "47948": {"logprob": -4.459475517272949, "rank": 5, "decoded_token": " stretching"}}, {"11223": {"logprob": -1.3947651386260986, "rank": 1, "decoded_token": " green"}, "24880": {"logprob": -1.8947651386260986, "rank": 2, "decoded_token": " varying"}, "95746": {"logprob": -2.0822651386260986, "rank": 3, "decoded_token": " rocky"}, "1295": {"logprob": -3.0197651386260986, "rank": 4, "decoded_token": " l"}, "19546": {"logprob": -3.0822651386260986, "rank": 5, "decoded_token": " varied"}}, {"1321": {"logprob": -0.8649212121963501, "rank": 1, "decoded_token": " and"}, "61263": {"logprob": -1.73992121219635, "rank": 2, "decoded_token": " slopes"}, "47260": {"logprob": -1.86492121219635, "rank": 3, "decoded_token": " vegetation"}, "50373": {"logprob": -1.98992121219635, "rank": 4, "decoded_token": " patches"}, "23170": {"logprob": -3.4899210929870605, "rank": 5, "decoded_token": " grass"}}, {"95746": {"logprob": -0.21662631630897522, "rank": 1, "decoded_token": " rocky"}, "22980": {"logprob": -1.9666262865066528, "rank": 2, "decoded_token": " brown"}, "26549": {"logprob": -3.8416264057159424, "rank": 3, "decoded_token": " gray"}, "4266": {"logprob": -4.216626167297363, "rank": 4, "decoded_token": " bar"}, "34052": {"logprob": -4.966626167297363, "rank": 5, "decoded_token": " grey"}}, {"24765": {"logprob": -0.32041722536087036, "rank": 1, "decoded_token": " terrain"}, "57912": {"logprob": -1.8204171657562256, "rank": 2, "decoded_token": " terrains"}, "61263": {"logprob": -2.6954171657562256, "rank": 3, "decoded_token": " slopes"}, "84497": {"logprob": -3.9454171657562256, "rank": 4, "decoded_token": " landscapes"}, "17764": {"logprob": -4.695417404174805, "rank": 5, "decoded_token": " surfaces"}}, {"2425": {"logprob": -0.4664109945297241, "rank": 1, "decoded_token": " under"}, "1046": {"logprob": -1.4664109945297241, "rank": 2, "decoded_token": "."}, "1044": {"logprob": -3.4664111137390137, "rank": 3, "decoded_token": ","}, "22923": {"logprob": -3.9664111137390137, "rank": 4, "decoded_token": " extending"}, "47948": {"logprob": -4.091411113739014, "rank": 5, "decoded_token": " stretching"}}, {"1261": {"logprob": -0.015043734572827816, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -4.76504373550415, "rank": 2, "decoded_token": " an"}, "6133": {"logprob": -6.01504373550415, "rank": 3, "decoded_token": " clear"}, "1278": {"logprob": -6.26504373550415, "rank": 4, "decoded_token": " the"}, "16152": {"logprob": -7.26504373550415, "rank": 5, "decoded_token": " cloud"}}, {"6133": {"logprob": -0.7420746684074402, "rank": 1, "decoded_token": " clear"}, "18416": {"logprob": -1.492074728012085, "rank": 2, "decoded_token": " haz"}, "16152": {"logprob": -1.992074728012085, "rank": 3, "decoded_token": " cloud"}, "27254": {"logprob": -3.367074728012085, "rank": 4, "decoded_token": " partly"}, "4391": {"logprob": -3.617074728012085, "rank": 5, "decoded_token": " light"}}, {"21283": {"logprob": -0.007355513051152229, "rank": 1, "decoded_token": " sky"}, "10991": {"logprob": -5.257355690002441, "rank": 2, "decoded_token": " blue"}, "1044": {"logprob": -6.382355690002441, "rank": 3, "decoded_token": ","}, "1505": {"logprob": -8.257355690002441, "rank": 4, "decoded_token": " or"}, "3950": {"logprob": -10.132355690002441, "rank": 5, "decoded_token": " day"}}, {"1046": {"logprob": -0.01126158982515335, "rank": 1, "decoded_token": "."}, "1626": {"logprob": -4.636261463165283, "rank": 2, "decoded_token": ".\n"}, "1338": {"logprob": -7.761261463165283, "rank": 3, "decoded_token": ".\n\n"}, "1044": {"logprob": -7.761261463165283, "rank": 4, "decoded_token": ","}, "1395": {"logprob": -8.011261940002441, "rank": 5, "decoded_token": " is"}}, {"2": {"logprob": -0.00709608756005764, "rank": 1, "decoded_token": " "}, "1032": {"logprob": -5.007096290588379, "rank": 2, "decoded_token": " The"}, "1256": {"logprob": -8.132096290588379, "rank": 3, "decoded_token": " "}}]], [[1049, 1046, 1349, 7244, 10575, 1395, 28528, 1408, 1261, 32656, 11237, 1044, 7283, 2015, 1513, 1278, 13424, 1626, 1050, 1046, 1349, 122203, 24361, 28035, 1454, 11223, 1321, 95746, 24765, 2425, 1261, 6133, 21283, 1626, 1051, 1046, 1349, 2965, 1294, 1261, 4804, 4250, 12006, 4302, 48049, 4837, 1261, 29397, 1435, 22140, 21457, 22196, 1626, 1052, 1046, 1349, 53301, 59396, 3549, 1294, 1261, 12097, 1044, 121040, 1536, 11223, 23170, 1321, 17744, 34941, 16429, 2425, 1261, 10991, 21283, 1046, 2], "1. A black dog is lying on a wooden floor, looking up at the camera.\n2. A rugged mountain landscape with green and rocky terrain under a clear sky.\n3. A person in a red swimsuit walks along a beach as waves crash nearby.\n4. A winding gravel path in a park, bordered by green grass and blooming trees under a blue sky.", [{"1049": {"logprob": -0.17000193893909454, "rank": 1, "decoded_token": "1"}, "11745": {"logprob": -1.9200019836425781, "rank": 2, "decoded_token": "Here"}, "69957": {"logprob": -4.920001983642578, "rank": 3, "decoded_token": "Sure"}, "117991": {"logprob": -7.295001983642578, "rank": 4, "decoded_token": "Certain"}, "1784": {"logprob": -7.295001983642578, "rank": 5, "decoded_token": "The"}}, {"1046": {"logprob": -1.597391747054644e-05, "rank": 1, "decoded_token": "."}, "1041": {"logprob": -11.500016212463379, "rank": 2, "decoded_token": ")"}, "1058": {"logprob": -13.062516212463379, "rank": 3, "decoded_token": ":"}, "3590": {"logprob": -13.750016212463379, "rank": 4, "decoded_token": ".A"}, "48426": {"logprob": -14.312516212463379, "rank": 5, "decoded_token": ".The"}}, {"1349": {"logprob": -0.07567699253559113, "rank": 1, "decoded_token": " A"}, "1531": {"logprob": -3.075676918029785, "rank": 2, "decoded_token": " The"}, "1603": {"logprob": -3.950676918029785, "rank": 3, "decoded_token": " **"}, "2409": {"logprob": -6.075676918029785, "rank": 4, "decoded_token": " This"}, "8479": {"logprob": -6.575676918029785, "rank": 5, "decoded_token": " Black"}}, {"7244": {"logprob": -0.06906593590974808, "rank": 1, "decoded_token": " black"}, "16450": {"logprob": -3.694066047668457, "rank": 2, "decoded_token": " sle"}, "6231": {"logprob": -4.506566047668457, "rank": 3, "decoded_token": " close"}, "4329": {"logprob": -4.944066047668457, "rank": 4, "decoded_token": " large"}, "8500": {"logprob": -5.256566047668457, "rank": 5, "decoded_token": " dark"}}, {"10575": {"logprob": -0.11913803219795227, "rank": 1, "decoded_token": " dog"}, "116572": {"logprob": -2.24413800239563, "rank": 2, "decoded_token": " puppy"}, "119075": {"logprob": -5.494138240814209, "rank": 3, "decoded_token": " Labrador"}, "28404": {"logprob": -7.181638240814209, "rank": 4, "decoded_token": " pup"}, "8636": {"logprob": -7.869138240814209, "rank": 5, "decoded_token": " lab"}}, {"1395": {"logprob": -0.782707154750824, "rank": 1, "decoded_token": " is"}, "22524": {"logprob": -1.1577072143554688, "rank": 2, "decoded_token": " lies"}, "1454": {"logprob": -2.9077072143554688, "rank": 3, "decoded_token": " with"}, "10637": {"logprob": -3.0327072143554688, "rank": 4, "decoded_token": " looks"}, "28528": {"logprob": -3.5327072143554688, "rank": 5, "decoded_token": " lying"}}, {"28528": {"logprob": -0.3443163335323334, "rank": 1, "decoded_token": " lying"}, "7283": {"logprob": -2.094316244125366, "rank": 2, "decoded_token": " looking"}, "60700": {"logprob": -2.844316244125366, "rank": 3, "decoded_token": " laying"}, "38235": {"logprob": -3.344316244125366, "rank": 4, "decoded_token": " resting"}, "18970": {"logprob": -3.469316244125366, "rank": 5, "decoded_token": " sitting"}}, {"1408": {"logprob": -0.29093095660209656, "rank": 1, "decoded_token": " on"}, "3151": {"logprob": -1.415930986404419, "rank": 2, "decoded_token": " down"}, "41132": {"logprob": -6.16593074798584, "rank": 3, "decoded_token": " attent"}, "1321": {"logprob": -6.85343074798584, "rank": 4, "decoded_token": " and"}, "14038": {"logprob": -6.97843074798584, "rank": 5, "decoded_token": " flat"}}, {"1261": {"logprob": -0.05553353577852249, "rank": 1, "decoded_token": " a"}, "2246": {"logprob": -3.6805336475372314, "rank": 2, "decoded_token": " its"}, "32656": {"logprob": -3.8055336475372314, "rank": 3, "decoded_token": " wooden"}, "1278": {"logprob": -5.305533409118652, "rank": 4, "decoded_token": " the"}, "3977": {"logprob": -7.430533409118652, "rank": 5, "decoded_token": " top"}}, {"32656": {"logprob": -0.039505477994680405, "rank": 1, "decoded_token": " wooden"}, "3403": {"logprob": -3.9145054817199707, "rank": 2, "decoded_token": " text"}, "44130": {"logprob": -4.414505481719971, "rank": 3, "decoded_token": " rust"}, "12603": {"logprob": -5.914505481719971, "rank": 4, "decoded_token": " wood"}, "17253": {"logprob": -6.539505481719971, "rank": 5, "decoded_token": " weather"}}, {"11237": {"logprob": -0.373188853263855, "rank": 1, "decoded_token": " floor"}, "4691": {"logprob": -1.248188853263855, "rank": 2, "decoded_token": " surface"}, "1615": {"logprob": -4.2481889724731445, "rank": 3, "decoded_token": " pl"}, "3403": {"logprob": -5.6231889724731445, "rank": 4, "decoded_token": " text"}, "28984": {"logprob": -5.9981889724731445, "rank": 5, "decoded_token": " deck"}}, {"1044": {"logprob": -1.378434181213379, "rank": 3, "decoded_token": ","}, "7283": {"logprob": -1.378434181213379, "rank": 1, "decoded_token": " looking"}, "1626": {"logprob": -1.378434181213379, "rank": 2, "decoded_token": ".\n"}, "1321": {"logprob": -2.378434181213379, "rank": 4, "decoded_token": " and"}, "1454": {"logprob": -2.628434181213379, "rank": 5, "decoded_token": " with"}}, {"7283": {"logprob": -0.17630912363529205, "rank": 1, "decoded_token": " looking"}, "11589": {"logprob": -2.551309108734131, "rank": 2, "decoded_token": " gaz"}, "35542": {"logprob": -2.676309108734131, "rank": 3, "decoded_token": " staring"}, "22116": {"logprob": -6.238809108734131, "rank": 4, "decoded_token": " facing"}, "11735": {"logprob": -6.488809108734131, "rank": 5, "decoded_token": " giving"}}, {"2015": {"logprob": -0.8436563014984131, "rank": 1, "decoded_token": " up"}, "7655": {"logprob": -1.343656301498413, "rank": 2, "decoded_token": " directly"}, "74606": {"logprob": -1.718656301498413, "rank": 3, "decoded_token": " upwards"}, "40022": {"logprob": -2.593656301498413, "rank": 4, "decoded_token": " upward"}, "11521": {"logprob": -4.406156539916992, "rank": 5, "decoded_token": " straight"}}, {"1513": {"logprob": -0.45780688524246216, "rank": 1, "decoded_token": " at"}, "1626": {"logprob": -1.7078068256378174, "rank": 2, "decoded_token": ".\n"}, "1454": {"logprob": -2.3328068256378174, "rank": 3, "decoded_token": " with"}, "1935": {"logprob": -3.5828068256378174, "rank": 4, "decoded_token": " int"}, "41132": {"logprob": -3.9578068256378174, "rank": 5, "decoded_token": " attent"}}, {"1278": {"logprob": -0.0004164305282756686, "rank": 1, "decoded_token": " the"}, "4433": {"logprob": -8.00041675567627, "rank": 2, "decoded_token": " something"}, "1261": {"logprob": -10.50041675567627, "rank": 3, "decoded_token": " a"}, "2246": {"logprob": -10.87541675567627, "rank": 4, "decoded_token": " its"}, "1636": {"logprob": -11.37541675567627, "rank": 5, "decoded_token": " you"}}, {"13424": {"logprob": -0.000399033073335886, "rank": 1, "decoded_token": " camera"}, "56268": {"logprob": -8.125398635864258, "rank": 2, "decoded_token": " viewer"}, "68439": {"logprob": -9.500398635864258, "rank": 3, "decoded_token": " photographer"}, "37967": {"logprob": -12.000398635864258, "rank": 4, "decoded_token": " ceiling"}, "2965": {"logprob": -12.312898635864258, "rank": 5, "decoded_token": " person"}}, {"1626": {"logprob": -0.10298559814691544, "rank": 1, "decoded_token": ".\n"}, "1046": {"logprob": -2.9779856204986572, "rank": 2, "decoded_token": "."}, "1454": {"logprob": -3.2279856204986572, "rank": 3, "decoded_token": " with"}, "1338": {"logprob": -5.227985382080078, "rank": 4, "decoded_token": ".\n\n"}, "1935": {"logprob": -6.852985382080078, "rank": 5, "decoded_token": " int"}}, {"1050": {"logprob": -0.002897590398788452, "rank": 1, "decoded_token": "2"}, "1256": {"logprob": -6.5028977394104, "rank": 2, "decoded_token": " "}, "1293": {"logprob": -6.6278977394104, "rank": 3, "decoded_token": " "}, "1032": {"logprob": -9.877897262573242, "rank": 4, "decoded_token": " "}, "1009": {"logprob": -11.627897262573242, "rank": 5, "decoded_token": "\t"}}, {"1046": {"logprob": -1.5497195136049413e-06, "rank": 1, "decoded_token": "."}, "1044": {"logprob": -14.875001907348633, "rank": 2, "decoded_token": ","}, "3590": {"logprob": -15.000001907348633, "rank": 3, "decoded_token": ".A"}, "2247": {"logprob": -15.125001907348633, "rank": 4, "decoded_token": " ."}, "1058": {"logprob": -15.375001907348633, "rank": 5, "decoded_token": ":"}}, {"1349": {"logprob": -0.6107801198959351, "rank": 1, "decoded_token": " A"}, "11826": {"logprob": -1.360780119895935, "rank": 2, "decoded_token": " Maj"}, "113465": {"logprob": -2.3607802391052246, "rank": 3, "decoded_token": " Rug"}, "27260": {"logprob": -3.7357802391052246, "rank": 4, "decoded_token": " Mountain"}, "1531": {"logprob": -4.485780239105225, "rank": 5, "decoded_token": " The"}}, {"122203": {"logprob": -0.8547073602676392, "rank": 1, "decoded_token": " rugged"}, "15375": {"logprob": -2.1047072410583496, "rank": 2, "decoded_token": " vast"}, "10726": {"logprob": -2.1047072410583496, "rank": 3, "decoded_token": " scen"}, "61082": {"logprob": -2.6047072410583496, "rank": 4, "decoded_token": " panor"}, "2965": {"logprob": -3.2922072410583496, "rank": 5, "decoded_token": " person"}}, {"24361": {"logprob": -0.41217130422592163, "rank": 1, "decoded_token": " mountain"}, "1044": {"logprob": -1.6621713638305664, "rank": 2, "decoded_token": ","}, "127945": {"logprob": -2.6621713638305664, "rank": 3, "decoded_token": " mountainous"}, "28035": {"logprob": -3.5371713638305664, "rank": 4, "decoded_token": " landscape"}, "1321": {"logprob": -3.6621713638305664, "rank": 5, "decoded_token": " and"}}, {"28035": {"logprob": -0.6676621437072754, "rank": 1, "decoded_token": " landscape"}, "4521": {"logprob": -0.7926621437072754, "rank": 2, "decoded_token": " range"}, "24765": {"logprob": -4.542662143707275, "rank": 3, "decoded_token": " terrain"}, "13327": {"logprob": -5.167662143707275, "rank": 4, "decoded_token": " scene"}, "12248": {"logprob": -5.167662143707275, "rank": 5, "decoded_token": " peak"}}, {"1454": {"logprob": -0.31015345454216003, "rank": 1, "decoded_token": " with"}, "6971": {"logprob": -2.4351534843444824, "rank": 2, "decoded_token": " features"}, "94973": {"logprob": -3.3101534843444824, "rank": 3, "decoded_token": " stretches"}, "89995": {"logprob": -3.4351534843444824, "rank": 4, "decoded_token": " showc"}, "1395": {"logprob": -3.5601534843444824, "rank": 5, "decoded_token": " is"}}, {"11223": {"logprob": -1.547694206237793, "rank": 1, "decoded_token": " green"}, "95746": {"logprob": -1.922694206237793, "rank": 2, "decoded_token": " rocky"}, "27469": {"logprob": -2.172694206237793, "rank": 3, "decoded_token": " peaks"}, "6245": {"logprob": -2.297694206237793, "rank": 4, "decoded_token": " multiple"}, "47147": {"logprob": -2.360194206237793, "rank": 5, "decoded_token": " steep"}}, {"1321": {"logprob": -0.9617817401885986, "rank": 1, "decoded_token": " and"}, "61263": {"logprob": -1.3367817401885986, "rank": 2, "decoded_token": " slopes"}, "51187": {"logprob": -2.3367817401885986, "rank": 3, "decoded_token": " hills"}, "47260": {"logprob": -2.3367817401885986, "rank": 4, "decoded_token": " vegetation"}, "50373": {"logprob": -2.7117817401885986, "rank": 5, "decoded_token": " patches"}}, {"95746": {"logprob": -0.11686273664236069, "rank": 1, "decoded_token": " rocky"}, "22980": {"logprob": -2.7418627738952637, "rank": 2, "decoded_token": " brown"}, "4266": {"logprob": -3.8668627738952637, "rank": 3, "decoded_token": " bar"}, "26549": {"logprob": -4.491862773895264, "rank": 4, "decoded_token": " gray"}, "9091": {"logprob": -5.366862773895264, "rank": 5, "decoded_token": " rock"}}, {"24765": {"logprob": -0.22640009224414825, "rank": 1, "decoded_token": " terrain"}, "57912": {"logprob": -2.476400136947632, "rank": 2, "decoded_token": " terrains"}, "61263": {"logprob": -2.726400136947632, "rank": 3, "decoded_token": " slopes"}, "51187": {"logprob": -3.851400136947632, "rank": 4, "decoded_token": " hills"}, "27469": {"logprob": -3.976400136947632, "rank": 5, "decoded_token": " peaks"}}, {"2425": {"logprob": -0.7823817133903503, "rank": 1, "decoded_token": " under"}, "1626": {"logprob": -1.1573817729949951, "rank": 2, "decoded_token": ".\n"}, "94973": {"logprob": -2.657381772994995, "rank": 3, "decoded_token": " stretches"}, "1395": {"logprob": -2.782381772994995, "rank": 4, "decoded_token": " is"}, "7038": {"logprob": -3.532381772994995, "rank": 5, "decoded_token": " extends"}}, {"1261": {"logprob": -0.016132064163684845, "rank": 1, "decoded_token": " a"}, "6133": {"logprob": -5.39113187789917, "rank": 2, "decoded_token": " clear"}, "1420": {"logprob": -5.39113187789917, "rank": 3, "decoded_token": " an"}, "1278": {"logprob": -6.01613187789917, "rank": 4, "decoded_token": " the"}, "16152": {"logprob": -6.26613187789917, "rank": 5, "decoded_token": " cloud"}}, {"6133": {"logprob": -0.44541382789611816, "rank": 1, "decoded_token": " clear"}, "16152": {"logprob": -2.070413827896118, "rank": 2, "decoded_token": " cloud"}, "18416": {"logprob": -2.320413827896118, "rank": 3, "decoded_token": " haz"}, "27254": {"logprob": -3.195413827896118, "rank": 4, "decoded_token": " partly"}, "10991": {"logprob": -3.320413827896118, "rank": 5, "decoded_token": " blue"}}, {"21283": {"logprob": -0.003768961876630783, "rank": 1, "decoded_token": " sky"}, "10991": {"logprob": -5.7537689208984375, "rank": 2, "decoded_token": " blue"}, "1044": {"logprob": -7.6287689208984375, "rank": 3, "decoded_token": ","}, "1505": {"logprob": -10.753768920898438, "rank": 4, "decoded_token": " or"}, "3044": {"logprob": -11.128768920898438, "rank": 5, "decoded_token": " sk"}}, {"1626": {"logprob": -0.0008177988929674029, "rank": 1, "decoded_token": ".\n"}, "1046": {"logprob": -7.375817775726318, "rank": 2, "decoded_token": "."}, "1395": {"logprob": -9.750818252563477, "rank": 3, "decoded_token": " is"}, "1010": {"logprob": -10.125818252563477, "rank": 4, "decoded_token": "\n"}, "1044": {"logprob": -10.750818252563477, "rank": 5, "decoded_token": ","}}, {"1051": {"logprob": -0.00013457823661156, "rank": 1, "decoded_token": "3"}, "1052": {"logprob": -9.125134468078613, "rank": 2, "decoded_token": "4"}, "1256": {"logprob": -11.375134468078613, "rank": 3, "decoded_token": " "}, "1050": {"logprob": -11.875134468078613, "rank": 4, "decoded_token": "2"}, "1049": {"logprob": -13.000134468078613, "rank": 5, "decoded_token": "1"}}, {"1046": {"logprob": -7.152555099310121e-07, "rank": 1, "decoded_token": "."}, "3590": {"logprob": -14.875000953674316, "rank": 2, "decoded_token": ".A"}, "48426": {"logprob": -15.937500953674316, "rank": 3, "decoded_token": ".The"}, "1349": {"logprob": -17.0, "rank": 4, "decoded_token": " A"}, "1338": {"logprob": -17.3125, "rank": 5, "decoded_token": ".\n\n"}}, {"1349": {"logprob": -0.03193942829966545, "rank": 1, "decoded_token": " A"}, "10638": {"logprob": -4.406939506530762, "rank": 2, "decoded_token": " Two"}, "2048": {"logprob": -5.031939506530762, "rank": 3, "decoded_token": " An"}, "1488": {"logprob": -5.156939506530762, "rank": 4, "decoded_token": " W"}, "15035": {"logprob": -5.906939506530762, "rank": 5, "decoded_token": " People"}}, {"2965": {"logprob": -0.41655251383781433, "rank": 1, "decoded_token": " person"}, "92731": {"logprob": -1.5415525436401367, "rank": 2, "decoded_token": " lone"}, "79013": {"logprob": -2.7915525436401367, "rank": 3, "decoded_token": " solitary"}, "29397": {"logprob": -3.5415525436401367, "rank": 4, "decoded_token": " beach"}, "2169": {"logprob": -4.729052543640137, "rank": 5, "decoded_token": " ser"}}, {"1294": {"logprob": -0.9845026135444641, "rank": 1, "decoded_token": " in"}, "1395": {"logprob": -1.2345025539398193, "rank": 2, "decoded_token": " is"}, "48049": {"logprob": -1.8595025539398193, "rank": 3, "decoded_token": " walks"}, "23737": {"logprob": -2.2345025539398193, "rank": 4, "decoded_token": " stands"}, "1285": {"logprob": -2.8595025539398193, "rank": 5, "decoded_token": " w"}}, {"1261": {"logprob": -0.32012784481048584, "rank": 1, "decoded_token": " a"}, "4804": {"logprob": -1.3201278448104858, "rank": 2, "decoded_token": " red"}, "1420": {"logprob": -5.820127964019775, "rank": 3, "decoded_token": " an"}, "64031": {"logprob": -6.570127964019775, "rank": 4, "decoded_token": " swim"}, "18168": {"logprob": -6.695127964019775, "rank": 5, "decoded_token": " bright"}}, {"4804": {"logprob": -0.10999592393636703, "rank": 1, "decoded_token": " red"}, "1285": {"logprob": -2.3599958419799805, "rank": 2, "decoded_token": " w"}, "4250": {"logprob": -5.6099958419799805, "rank": 3, "decoded_token": " sw"}, "18168": {"logprob": -6.0474958419799805, "rank": 4, "decoded_token": " bright"}, "18258": {"logprob": -6.4224958419799805, "rank": 5, "decoded_token": " wet"}}, {"4250": {"logprob": -0.2469252496957779, "rank": 1, "decoded_token": " sw"}, "1285": {"logprob": -2.3719253540039062, "rank": 2, "decoded_token": " w"}, "64031": {"logprob": -2.7469253540039062, "rank": 3, "decoded_token": " swim"}, "17513": {"logprob": -3.2469253540039062, "rank": 4, "decoded_token": " suit"}, "75948": {"logprob": -4.371925354003906, "rank": 5, "decoded_token": " outfit"}}, {"12006": {"logprob": -5.722029527532868e-06, "rank": 1, "decoded_token": "ims"}, "25763": {"logprob": -12.750005722045898, "rank": 2, "decoded_token": "immer"}, "7552": {"logprob": -13.687505722045898, "rank": 3, "decoded_token": "imm"}, "2097": {"logprob": -16.6875057220459, "rank": 4, "decoded_token": "ins"}, "19523": {"logprob": -16.7500057220459, "rank": 5, "decoded_token": "imb"}}, {"4302": {"logprob": -1.8000440832111053e-05, "rank": 1, "decoded_token": "uit"}, "17513": {"logprob": -11.875018119812012, "rank": 2, "decoded_token": " suit"}, "8036": {"logprob": -13.250018119812012, "rank": 3, "decoded_token": "irt"}, "36953": {"logprob": -13.500018119812012, "rank": 4, "decoded_token": "uiten"}, "1276": {"logprob": -14.437518119812012, "rank": 5, "decoded_token": "it"}}, {"48049": {"logprob": -0.41766560077667236, "rank": 1, "decoded_token": " walks"}, "1395": {"logprob": -1.4176656007766724, "rank": 2, "decoded_token": " is"}, "19710": {"logprob": -2.792665481567383, "rank": 3, "decoded_token": " walking"}, "23737": {"logprob": -3.917665481567383, "rank": 4, "decoded_token": " stands"}, "1285": {"logprob": -4.292665481567383, "rank": 5, "decoded_token": " w"}}, {"4837": {"logprob": -0.002689199522137642, "rank": 1, "decoded_token": " along"}, "9412": {"logprob": -6.627689361572266, "rank": 2, "decoded_token": " alone"}, "6117": {"logprob": -7.377689361572266, "rank": 3, "decoded_token": " near"}, "1408": {"logprob": -8.002689361572266, "rank": 4, "decoded_token": " on"}, "2203": {"logprob": -8.377689361572266, "rank": 5, "decoded_token": " into"}}, {"1261": {"logprob": -0.38749611377716064, "rank": 1, "decoded_token": " a"}, "1278": {"logprob": -1.1374961137771606, "rank": 2, "decoded_token": " the"}, "1420": {"logprob": -7.387495994567871, "rank": 3, "decoded_token": " an"}, "100991": {"logprob": -13.949995994567871, "rank": 4, "decoded_token": " sandy"}, "18258": {"logprob": -14.512495994567871, "rank": 5, "decoded_token": " wet"}}, {"29397": {"logprob": -0.5292408466339111, "rank": 1, "decoded_token": " beach"}, "100991": {"logprob": -0.9042408466339111, "rank": 2, "decoded_token": " sandy"}, "1627": {"logprob": -6.029240608215332, "rank": 3, "decoded_token": " sh"}, "46422": {"logprob": -6.529240608215332, "rank": 4, "decoded_token": " shore"}, "2169": {"logprob": -7.779240608215332, "rank": 5, "decoded_token": " ser"}}, {"1435": {"logprob": -0.29965779185295105, "rank": 1, "decoded_token": " as"}, "1454": {"logprob": -1.6746578216552734, "rank": 2, "decoded_token": " with"}, "1513": {"logprob": -3.7996578216552734, "rank": 3, "decoded_token": " at"}, "3016": {"logprob": -3.7996578216552734, "rank": 4, "decoded_token": " while"}, "6117": {"logprob": -4.799657821655273, "rank": 5, "decoded_token": " near"}}, {"22140": {"logprob": -0.015346773900091648, "rank": 1, "decoded_token": " waves"}, "1261": {"logprob": -4.515347003936768, "rank": 2, "decoded_token": " a"}, "1278": {"logprob": -6.140347003936768, "rank": 3, "decoded_token": " the"}, "27208": {"logprob": -6.890347003936768, "rank": 4, "decoded_token": " ocean"}, "4329": {"logprob": -7.265347003936768, "rank": 5, "decoded_token": " large"}}, {"21457": {"logprob": -0.013234862126410007, "rank": 1, "decoded_token": " crash"}, "33168": {"logprob": -5.138235092163086, "rank": 2, "decoded_token": " gently"}, "10401": {"logprob": -5.950735092163086, "rank": 3, "decoded_token": " roll"}, "4323": {"logprob": -6.700735092163086, "rank": 4, "decoded_token": " break"}, "5125": {"logprob": -7.138235092163086, "rank": 5, "decoded_token": " approach"}}, {"22196": {"logprob": -0.060372594743967056, "rank": 1, "decoded_token": " nearby"}, "6117": {"logprob": -3.3103725910186768, "rank": 2, "decoded_token": " near"}, "1294": {"logprob": -4.435372829437256, "rank": 3, "decoded_token": " in"}, "25644": {"logprob": -6.310372829437256, "rank": 4, "decoded_token": " beside"}, "1321": {"logprob": -6.560372829437256, "rank": 5, "decoded_token": " and"}}, {"1626": {"logprob": -0.005290080793201923, "rank": 1, "decoded_token": ".\n"}, "1294": {"logprob": -6.5052900314331055, "rank": 2, "decoded_token": " in"}, "1044": {"logprob": -7.0052900314331055, "rank": 3, "decoded_token": ","}, "1321": {"logprob": -7.1302900314331055, "rank": 4, "decoded_token": " and"}, "1513": {"logprob": -7.2552900314331055, "rank": 5, "decoded_token": " at"}}, {"1052": {"logprob": -7.748573807475623e-06, "rank": 1, "decoded_token": "4"}, "1051": {"logprob": -12.562507629394531, "rank": 2, "decoded_token": "3"}, "1053": {"logprob": -13.125007629394531, "rank": 3, "decoded_token": "5"}, "1256": {"logprob": -14.125007629394531, "rank": 4, "decoded_token": " "}, "1049": {"logprob": -14.312507629394531, "rank": 5, "decoded_token": "1"}}, {"1046": {"logprob": -1.2993727978027891e-05, "rank": 1, "decoded_token": "."}, "1044": {"logprob": -12.62501335144043, "rank": 2, "decoded_token": ","}, "3590": {"logprob": -12.75001335144043, "rank": 3, "decoded_token": ".A"}, "1058": {"logprob": -13.00001335144043, "rank": 4, "decoded_token": ":"}, "2247": {"logprob": -13.37501335144043, "rank": 5, "decoded_token": " ."}}, {"1349": {"logprob": -0.00046957432641647756, "rank": 1, "decoded_token": " A"}, "2048": {"logprob": -8.250469207763672, "rank": 2, "decoded_token": " An"}, "1488": {"logprob": -10.125469207763672, "rank": 3, "decoded_token": " W"}, "2409": {"logprob": -10.375469207763672, "rank": 4, "decoded_token": " This"}, "12232": {"logprob": -10.500469207763672, "rank": 5, "decoded_token": " Gra"}}, {"53301": {"logprob": -0.35120296478271484, "rank": 1, "decoded_token": " winding"}, "59396": {"logprob": -1.8512029647827148, "rank": 2, "decoded_token": " gravel"}, "2169": {"logprob": -2.476202964782715, "rank": 3, "decoded_token": " ser"}, "54742": {"logprob": -3.851202964782715, "rank": 4, "decoded_token": " peaceful"}, "43536": {"logprob": -5.101202964782715, "rank": 5, "decoded_token": " curved"}}, {"59396": {"logprob": -0.2955280840396881, "rank": 1, "decoded_token": " gravel"}, "3549": {"logprob": -1.6705280542373657, "rank": 2, "decoded_token": " path"}, "14801": {"logprob": -2.7955281734466553, "rank": 3, "decoded_token": " pathway"}, "1044": {"logprob": -6.420527935028076, "rank": 4, "decoded_token": ","}, "18341": {"logprob": -6.670527935028076, "rank": 5, "decoded_token": " pathways"}}, {"3549": {"logprob": -0.03408379852771759, "rank": 1, "decoded_token": " path"}, "14801": {"logprob": -3.409083843231201, "rank": 2, "decoded_token": " pathway"}, "18341": {"logprob": -8.284083366394043, "rank": 3, "decoded_token": " pathways"}, "1505": {"logprob": -9.534083366394043, "rank": 4, "decoded_token": " or"}, "7368": {"logprob": -10.659083366394043, "rank": 5, "decoded_token": "path"}}, {"1294": {"logprob": -1.0857839584350586, "rank": 1, "decoded_token": " in"}, "13335": {"logprob": -1.4607839584350586, "rank": 2, "decoded_token": " leads"}, "2645": {"logprob": -1.9607839584350586, "rank": 3, "decoded_token": " through"}, "29817": {"logprob": -2.4607839584350586, "rank": 4, "decoded_token": " surrounded"}, "22416": {"logprob": -3.2107839584350586, "rank": 5, "decoded_token": " curves"}}, {"1261": {"logprob": -0.00011705666838679463, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -9.500117301940918, "rank": 2, "decoded_token": " an"}, "1278": {"logprob": -10.250117301940918, "rank": 3, "decoded_token": " the"}, "2549": {"logprob": -12.750117301940918, "rank": 4, "decoded_token": " what"}, "11223": {"logprob": -13.750117301940918, "rank": 5, "decoded_token": " green"}}, {"12097": {"logprob": -0.02791696786880493, "rank": 1, "decoded_token": " park"}, "2169": {"logprob": -4.65291690826416, "rank": 2, "decoded_token": " ser"}, "1295": {"logprob": -4.65291690826416, "rank": 3, "decoded_token": " l"}, "23170": {"logprob": -5.27791690826416, "rank": 4, "decoded_token": " grass"}, "26428": {"logprob": -6.52791690826416, "rank": 5, "decoded_token": " garden"}}, {"1044": {"logprob": -1.350893259048462, "rank": 1, "decoded_token": ","}, "1395": {"logprob": -1.600893259048462, "rank": 2, "decoded_token": " is"}, "29817": {"logprob": -2.350893259048462, "rank": 3, "decoded_token": " surrounded"}, "121313": {"logprob": -2.475893259048462, "rank": 4, "decoded_token": " flanked"}, "1454": {"logprob": -2.475893259048462, "rank": 5, "decoded_token": " with"}}, {"121040": {"logprob": -0.710591197013855, "rank": 1, "decoded_token": " bordered"}, "121313": {"logprob": -1.085591197013855, "rank": 2, "decoded_token": " flanked"}, "54410": {"logprob": -1.960591197013855, "rank": 3, "decoded_token": " lined"}, "29817": {"logprob": -3.8355913162231445, "rank": 4, "decoded_token": " surrounded"}, "1454": {"logprob": -5.8355913162231445, "rank": 5, "decoded_token": " with"}}, {"1536": {"logprob": -4.6491513785440475e-06, "rank": 1, "decoded_token": " by"}, "1454": {"logprob": -12.375004768371582, "rank": 2, "decoded_token": " with"}, "1408": {"logprob": -15.812504768371582, "rank": 3, "decoded_token": " on"}, "3326": {"logprob": -16.875003814697266, "rank": 4, "decoded_token": "by"}, "1295": {"logprob": -16.875003814697266, "rank": 5, "decoded_token": " l"}}, {"11223": {"logprob": -0.4314780533313751, "rank": 1, "decoded_token": " green"}, "1295": {"logprob": -1.4314780235290527, "rank": 2, "decoded_token": " l"}, "23170": {"logprob": -2.4314780235290527, "rank": 3, "decoded_token": " grass"}, "17744": {"logprob": -4.806478023529053, "rank": 4, "decoded_token": " blo"}, "95612": {"logprob": -5.181478023529053, "rank": 5, "decoded_token": " vibrant"}}, {"23170": {"logprob": -0.00035041390219703317, "rank": 1, "decoded_token": " grass"}, "69230": {"logprob": -8.125349998474121, "rank": 2, "decoded_token": " lawn"}, "128633": {"logprob": -10.750349998474121, "rank": 3, "decoded_token": " grasses"}, "87781": {"logprob": -11.437849998474121, "rank": 4, "decoded_token": "\u8349"}, "16429": {"logprob": -11.437849998474121, "rank": 5, "decoded_token": " trees"}}, {"1321": {"logprob": -0.0009494088008068502, "rank": 1, "decoded_token": " and"}, "1044": {"logprob": -7.125949382781982, "rank": 2, "decoded_token": ","}, "1454": {"logprob": -9.25094985961914, "rank": 3, "decoded_token": " with"}, "2425": {"logprob": -11.75094985961914, "rank": 4, "decoded_token": " under"}, "1046": {"logprob": -11.75094985961914, "rank": 5, "decoded_token": "."}}, {"17744": {"logprob": -0.21488544344902039, "rank": 1, "decoded_token": " blo"}, "105368": {"logprob": -1.8398854732513428, "rank": 2, "decoded_token": " bloss"}, "87833": {"logprob": -3.8398854732513428, "rank": 3, "decoded_token": " flowering"}, "16429": {"logprob": -4.464885234832764, "rank": 4, "decoded_token": " trees"}, "117207": {"logprob": -7.589885234832764, "rank": 5, "decoded_token": " bloom"}}, {"34941": {"logprob": -7.152555099310121e-07, "rank": 1, "decoded_token": "oming"}, "35974": {"logprob": -14.375000953674316, "rank": 2, "decoded_token": "omed"}, "6325": {"logprob": -16.5625, "rank": 3, "decoded_token": "oms"}, "11009": {"logprob": -17.625, "rank": 4, "decoded_token": "omy"}, "9457": {"logprob": -18.875, "rank": 5, "decoded_token": "ming"}}, {"16429": {"logprob": -0.002424398437142372, "rank": 1, "decoded_token": " trees"}, "103796": {"logprob": -6.627424240112305, "rank": 2, "decoded_token": " cherry"}, "32152": {"logprob": -7.377424240112305, "rank": 3, "decoded_token": " flowers"}, "29151": {"logprob": -9.314924240112305, "rank": 4, "decoded_token": " shr"}, "20370": {"logprob": -9.564924240112305, "rank": 5, "decoded_token": " fruit"}}, {"2425": {"logprob": -0.3792523741722107, "rank": 1, "decoded_token": " under"}, "1046": {"logprob": -1.3792524337768555, "rank": 2, "decoded_token": "."}, "3675": {"logprob": -2.8792524337768555, "rank": 3, "decoded_token": " against"}, "1044": {"logprob": -5.1292524337768555, "rank": 4, "decoded_token": ","}, "1454": {"logprob": -7.2542524337768555, "rank": 5, "decoded_token": " with"}}, {"1261": {"logprob": -0.0002315968304174021, "rank": 1, "decoded_token": " a"}, "1278": {"logprob": -8.875231742858887, "rank": 2, "decoded_token": " the"}, "10991": {"logprob": -9.875231742858887, "rank": 3, "decoded_token": " blue"}, "6133": {"logprob": -10.375231742858887, "rank": 4, "decoded_token": " clear"}, "1420": {"logprob": -12.250231742858887, "rank": 5, "decoded_token": " an"}}, {"10991": {"logprob": -0.6372600197792053, "rank": 1, "decoded_token": " blue"}, "6133": {"logprob": -0.7622600197792053, "rank": 2, "decoded_token": " clear"}, "18168": {"logprob": -5.3872599601745605, "rank": 3, "decoded_token": " bright"}, "105573": {"logprob": -10.012260437011719, "rank": 4, "decoded_token": " sunny"}, "15330": {"logprob": -11.512260437011719, "rank": 5, "decoded_token": " Blue"}}, {"21283": {"logprob": -6.12716976320371e-05, "rank": 1, "decoded_token": " sky"}, "1044": {"logprob": -9.87506103515625, "rank": 2, "decoded_token": ","}, "19673": {"logprob": -12.00006103515625, "rank": 3, "decoded_token": " Sky"}, "1321": {"logprob": -13.31256103515625, "rank": 4, "decoded_token": " and"}, "124968": {"logprob": -14.81256103515625, "rank": 5, "decoded_token": " skies"}}, {"1046": {"logprob": -0.00013982271775603294, "rank": 1, "decoded_token": "."}, "2": {"logprob": -9.500140190124512, "rank": 2, "decoded_token": ".\n"}, "1626": {"logprob": -10.000140190124512, "rank": 3, "decoded_token": ".\n\n"}, "1338": {"logprob": -11.750140190124512, "rank": 4, "decoded_token": " with"}}, {"2": {"logprob": -0.0004533693427219987, "rank": 1, "decoded_token": " "}, "1032": {"logprob": -7.750453472137451, "rank": 2, "decoded_token": " Each"}, "1256": {"logprob": -11.125452995300293, "rank": 3, "decoded_token": " This"}}]]]
\ No newline at end of file
diff --git a/asian_ai/performance/vLLM/tests/models/fixtures/musicflamingo/expected_results_batched.json b/asian_ai/performance/vLLM/tests/models/fixtures/musicflamingo/expected_results_batched.json
new file mode 100644
index 0000000000000000000000000000000000000000..3f9134951d3818b64da5bfd1f39ecb9662362474
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/fixtures/musicflamingo/expected_results_batched.json
@@ -0,0 +1 @@
+{"transcriptions": ["This track is an energetic Eurodance / Dance‑Pop anthem that blends the bright, melodic sensibilities of mainstream pop with the driving, club‑ready pulse of classic Eurodance. The duration of the piece is ", "**Verse 1**\nMidnight cravings in bloom, lights flicker in the room, pepperoni dreams arise, pizza party on your skies\n\n**Verse 2**\nCheese melts on the crust, in flavor we trust, boxes stacked to the"], "token_ids": [[1986, 3754, 374, 458, 44855, 19461, 98875, 378, 107, 14, 378, 107, 35, 681, 55964, 11598, 55564, 429, 57843, 279, 9906, 11, 10581, 52760, 6097, 13450, 315, 20729, 2420, 448, 279, 9842, 11, 6335, 55964, 2307, 27235, 315, 11416, 19461, 98875, 13, 220, 576, 8090, 315, 279, 6573, 374, 220], [334, 68043, 220, 16, 1019, 33648, 9287, 88828, 304, 51454, 11, 12711, 28347, 261, 304, 279, 3054, 11, 24353, 20783, 18707, 30789, 11, 22502, 4614, 389, 697, 49293, 271, 334, 68043, 220, 17, 1019, 26843, 2367, 98091, 389, 279, 39612, 11, 304, 17172, 582, 6950, 11, 14697, 41315, 311, 279]]}
diff --git a/asian_ai/performance/vLLM/tests/models/fixtures/musicflamingo/expected_results_single.json b/asian_ai/performance/vLLM/tests/models/fixtures/musicflamingo/expected_results_single.json
new file mode 100644
index 0000000000000000000000000000000000000000..f6005745cdf1974c9d09a96c509b023585cfca80
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/fixtures/musicflamingo/expected_results_single.json
@@ -0,0 +1 @@
+{"transcriptions": ["This track is an energetic Eurodance / Dance‑Pop anthem that blends the bright, melodic sensibilities of mainstream pop with the driving, club‑ready pulse of classic Eurodance. The duration of the piece is "], "token_ids": [[1986, 3754, 374, 458, 44855, 19461, 98875, 378, 107, 14, 378, 107, 35, 681, 55964, 11598, 55564, 429, 57843, 279, 9906, 11, 10581, 52760, 6097, 13450, 315, 20729, 2420, 448, 279, 9842, 11, 6335, 55964, 2307, 27235, 315, 11416, 19461, 98875, 13, 220, 576, 8090, 315, 279, 6573, 374, 220]]}
diff --git a/asian_ai/performance/vLLM/tests/models/fixtures/pixtral_chat.json b/asian_ai/performance/vLLM/tests/models/fixtures/pixtral_chat.json
new file mode 100644
index 0000000000000000000000000000000000000000..643afb83d29b89894314d97eb2323986a6a77135
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/fixtures/pixtral_chat.json
@@ -0,0 +1 @@
+[[[1784, 3937, 6122, 1261, 7244, 10575, 18970, 1408, 1261, 32656, 4691, 1046, 2], "The image shows a black dog sitting on a wooden surface.", [{"1784": {"logprob": -0.11687260121107101, "rank": 1, "decoded_token": "The"}, "4380": {"logprob": -2.366872549057007, "rank": 2, "decoded_token": "This"}, "1049": {"logprob": -4.741872787475586, "rank": 3, "decoded_token": "1"}, "117991": {"logprob": -5.991872787475586, "rank": 4, "decoded_token": "Certain"}, "1785": {"logprob": -5.991872787475586, "rank": 5, "decoded_token": "In"}}, {"3937": {"logprob": -0.28887900710105896, "rank": 1, "decoded_token": " image"}, "2158": {"logprob": -1.4138790369033813, "rank": 2, "decoded_token": " first"}, "3977": {"logprob": -5.788878917694092, "rank": 3, "decoded_token": " top"}, "7244": {"logprob": -6.163878917694092, "rank": 4, "decoded_token": " black"}, "8061": {"logprob": -6.788878917694092, "rank": 5, "decoded_token": " images"}}, {"6122": {"logprob": -0.9653709530830383, "rank": 1, "decoded_token": " shows"}, "51948": {"logprob": -1.4653708934783936, "rank": 2, "decoded_token": " depicts"}, "6971": {"logprob": -1.4653708934783936, "rank": 3, "decoded_token": " features"}, "25981": {"logprob": -2.8403708934783936, "rank": 4, "decoded_token": " displays"}, "8688": {"logprob": -2.8403708934783936, "rank": 5, "decoded_token": " contains"}}, {"1261": {"logprob": -0.003059827256947756, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -6.2530598640441895, "rank": 2, "decoded_token": " an"}, "2295": {"logprob": -7.8780598640441895, "rank": 3, "decoded_token": " two"}, "2342": {"logprob": -7.8780598640441895, "rank": 4, "decoded_token": " only"}, "1278": {"logprob": -8.628059387207031, "rank": 5, "decoded_token": " the"}}, {"7244": {"logprob": -0.17616479098796844, "rank": 1, "decoded_token": " black"}, "6231": {"logprob": -2.3011648654937744, "rank": 2, "decoded_token": " close"}, "4249": {"logprob": -3.4261648654937744, "rank": 3, "decoded_token": " single"}, "4329": {"logprob": -5.113664627075195, "rank": 4, "decoded_token": " large"}, "10575": {"logprob": -5.176164627075195, "rank": 5, "decoded_token": " dog"}}, {"10575": {"logprob": -0.10940006375312805, "rank": 1, "decoded_token": " dog"}, "116572": {"logprob": -2.4844000339508057, "rank": 2, "decoded_token": " puppy"}, "119075": {"logprob": -4.109400272369385, "rank": 3, "decoded_token": " Labrador"}, "15812": {"logprob": -7.296900272369385, "rank": 4, "decoded_token": " Lab"}, "7990": {"logprob": -7.421900272369385, "rank": 5, "decoded_token": " cat"}}, {"18970": {"logprob": -0.8322296738624573, "rank": 1, "decoded_token": " sitting"}, "1454": {"logprob": -1.5822296142578125, "rank": 2, "decoded_token": " with"}, "28528": {"logprob": -1.9572296142578125, "rank": 3, "decoded_token": " lying"}, "7283": {"logprob": -2.2072296142578125, "rank": 4, "decoded_token": " looking"}, "15866": {"logprob": -3.0197296142578125, "rank": 5, "decoded_token": " standing"}}, {"1408": {"logprob": -0.08769982308149338, "rank": 1, "decoded_token": " on"}, "1321": {"logprob": -3.7126998901367188, "rank": 2, "decoded_token": " and"}, "3675": {"logprob": -3.9626998901367188, "rank": 3, "decoded_token": " against"}, "41132": {"logprob": -4.587699890136719, "rank": 4, "decoded_token": " attent"}, "1454": {"logprob": -5.087699890136719, "rank": 5, "decoded_token": " with"}}, {"1261": {"logprob": -0.5400654673576355, "rank": 1, "decoded_token": " a"}, "32656": {"logprob": -0.9150654673576355, "rank": 2, "decoded_token": " wooden"}, "3977": {"logprob": -5.415065288543701, "rank": 3, "decoded_token": " top"}, "12603": {"logprob": -5.540065288543701, "rank": 4, "decoded_token": " wood"}, "44130": {"logprob": -6.290065288543701, "rank": 5, "decoded_token": " rust"}}, {"32656": {"logprob": -0.02516966126859188, "rank": 1, "decoded_token": " wooden"}, "44130": {"logprob": -4.400169849395752, "rank": 2, "decoded_token": " rust"}, "12603": {"logprob": -5.275169849395752, "rank": 3, "decoded_token": " wood"}, "3403": {"logprob": -5.525169849395752, "rank": 4, "decoded_token": " text"}, "17253": {"logprob": -6.962669849395752, "rank": 5, "decoded_token": " weather"}}, {"4691": {"logprob": -0.7264319658279419, "rank": 1, "decoded_token": " surface"}, "11237": {"logprob": -0.8514319658279419, "rank": 2, "decoded_token": " floor"}, "7042": {"logprob": -2.6014318466186523, "rank": 3, "decoded_token": " background"}, "28984": {"logprob": -5.226431846618652, "rank": 4, "decoded_token": " deck"}, "1615": {"logprob": -5.726431846618652, "rank": 5, "decoded_token": " pl"}}, {"1046": {"logprob": -0.4668232202529907, "rank": 1, "decoded_token": "."}, "1044": {"logprob": -1.9668232202529907, "rank": 2, "decoded_token": ","}, "1321": {"logprob": -2.466823101043701, "rank": 3, "decoded_token": " and"}, "7283": {"logprob": -2.716823101043701, "rank": 4, "decoded_token": " looking"}, "1454": {"logprob": -2.716823101043701, "rank": 5, "decoded_token": " with"}}, {"2": {"logprob": -0.002247072057798505, "rank": 1, "decoded_token": ""}, "1531": {"logprob": -6.627246856689453, "rank": 2, "decoded_token": " The"}, "1032": {"logprob": -7.127246856689453, "rank": 3, "decoded_token": " "}, "3730": {"logprob": -9.877246856689453, "rank": 4, "decoded_token": " There"}, "1256": {"logprob": -11.127246856689453, "rank": 5, "decoded_token": " "}}]], [[1049, 1046, 1349, 7244, 10575, 1454, 2327, 94766, 32961, 53048, 41132, 3923, 1408, 1261, 32656, 4691, 1626, 1050, 1046, 1349, 15375, 24361, 4521, 1454, 122203, 27469, 94973, 2425, 1261, 16152, 1121, 21283, 1046, 2], "1. A black dog with floppy ears sits attentively on a wooden surface.\n2. A vast mountain range with rugged peaks stretches under a cloudy sky.", [{"1049": {"logprob": -0.42824622988700867, "rank": 1, "decoded_token": "1"}, "1045": {"logprob": -1.553246259689331, "rank": 2, "decoded_token": "-"}, "1065": {"logprob": -2.428246259689331, "rank": 3, "decoded_token": "A"}, "1784": {"logprob": -4.053246021270752, "rank": 4, "decoded_token": "The"}, "69957": {"logprob": -4.428246021270752, "rank": 5, "decoded_token": "Sure"}}, {"1046": {"logprob": -1.9788545614574105e-05, "rank": 1, "decoded_token": "."}, "1058": {"logprob": -11.750020027160645, "rank": 2, "decoded_token": ":"}, "3590": {"logprob": -12.125020027160645, "rank": 3, "decoded_token": ".A"}, "1065": {"logprob": -13.062520027160645, "rank": 4, "decoded_token": "A"}, "1041": {"logprob": -13.750020027160645, "rank": 5, "decoded_token": ")"}}, {"1349": {"logprob": -0.14020134508609772, "rank": 1, "decoded_token": " A"}, "1429": {"logprob": -2.3902013301849365, "rank": 2, "decoded_token": " \""}, "1603": {"logprob": -3.7652013301849365, "rank": 3, "decoded_token": " **"}, "11967": {"logprob": -4.890201568603516, "rank": 4, "decoded_token": " Image"}, "1531": {"logprob": -5.015201568603516, "rank": 5, "decoded_token": " The"}}, {"7244": {"logprob": -0.2003599852323532, "rank": 1, "decoded_token": " black"}, "38462": {"logprob": -3.075360059738159, "rank": 2, "decoded_token": " curious"}, "68076": {"logprob": -3.575360059738159, "rank": 3, "decoded_token": " cute"}, "4329": {"logprob": -3.887860059738159, "rank": 4, "decoded_token": " large"}, "6231": {"logprob": -4.32535982131958, "rank": 5, "decoded_token": " close"}}, {"10575": {"logprob": -0.18818901479244232, "rank": 1, "decoded_token": " dog"}, "116572": {"logprob": -2.0631890296936035, "rank": 2, "decoded_token": " puppy"}, "119075": {"logprob": -3.1881890296936035, "rank": 3, "decoded_token": " Labrador"}, "15812": {"logprob": -6.9381890296936035, "rank": 4, "decoded_token": " Lab"}, "8636": {"logprob": -7.3131890296936035, "rank": 5, "decoded_token": " lab"}}, {"1454": {"logprob": -0.5699259042739868, "rank": 1, "decoded_token": " with"}, "53048": {"logprob": -1.2574259042739868, "rank": 2, "decoded_token": " sits"}, "1395": {"logprob": -3.0699257850646973, "rank": 3, "decoded_token": " is"}, "22524": {"logprob": -3.6324257850646973, "rank": 4, "decoded_token": " lies"}, "18970": {"logprob": -3.7574257850646973, "rank": 5, "decoded_token": " sitting"}}, {"2327": {"logprob": -1.2377738952636719, "rank": 1, "decoded_token": " fl"}, "1261": {"logprob": -1.3627738952636719, "rank": 2, "decoded_token": " a"}, "17300": {"logprob": -1.9252738952636719, "rank": 3, "decoded_token": " soul"}, "100089": {"logprob": -2.675273895263672, "rank": 4, "decoded_token": " expressive"}, "6444": {"logprob": -3.237773895263672, "rank": 5, "decoded_token": " soft"}}, {"94766": {"logprob": -0.0025601964443922043, "rank": 1, "decoded_token": "oppy"}, "124603": {"logprob": -6.315060138702393, "rank": 2, "decoded_token": "uffy"}, "1484": {"logprob": -7.877560138702393, "rank": 3, "decoded_token": "op"}, "24897": {"logprob": -8.81506061553955, "rank": 4, "decoded_token": "appy"}, "102477": {"logprob": -9.69006061553955, "rank": 5, "decoded_token": "opping"}}, {"32961": {"logprob": -5.113947918289341e-05, "rank": 1, "decoded_token": " ears"}, "16962": {"logprob": -11.250051498413086, "rank": 2, "decoded_token": " ear"}, "5731": {"logprob": -11.812551498413086, "rank": 3, "decoded_token": " eyes"}, "3351": {"logprob": -12.000051498413086, "rank": 4, "decoded_token": " years"}, "42071": {"logprob": -13.062551498413086, "rank": 5, "decoded_token": " cheeks"}}, {"53048": {"logprob": -0.6179640889167786, "rank": 1, "decoded_token": " sits"}, "10637": {"logprob": -1.9929640293121338, "rank": 2, "decoded_token": " looks"}, "1321": {"logprob": -2.430464029312134, "rank": 3, "decoded_token": " and"}, "1395": {"logprob": -2.617964029312134, "rank": 4, "decoded_token": " is"}, "18970": {"logprob": -3.055464029312134, "rank": 5, "decoded_token": " sitting"}}, {"41132": {"logprob": -0.3746516704559326, "rank": 1, "decoded_token": " attent"}, "1408": {"logprob": -2.3121516704559326, "rank": 2, "decoded_token": " on"}, "106534": {"logprob": -2.3746516704559326, "rank": 3, "decoded_token": " calmly"}, "12276": {"logprob": -2.6246516704559326, "rank": 4, "decoded_token": " alert"}, "6482": {"logprob": -5.124651908874512, "rank": 5, "decoded_token": " patient"}}, {"3923": {"logprob": -8.463501580990851e-05, "rank": 1, "decoded_token": "ively"}, "1556": {"logprob": -9.50008487701416, "rank": 2, "decoded_token": "ive"}, "6655": {"logprob": -11.87508487701416, "rank": 3, "decoded_token": "atively"}, "3929": {"logprob": -14.00008487701416, "rank": 4, "decoded_token": "ently"}, "47885": {"logprob": -14.62508487701416, "rank": 5, "decoded_token": "edly"}}, {"1408": {"logprob": -0.06439964473247528, "rank": 1, "decoded_token": " on"}, "3675": {"logprob": -3.0643997192382812, "rank": 2, "decoded_token": " against"}, "1294": {"logprob": -4.939399719238281, "rank": 3, "decoded_token": " in"}, "7283": {"logprob": -5.689399719238281, "rank": 4, "decoded_token": " looking"}, "1044": {"logprob": -5.814399719238281, "rank": 5, "decoded_token": ","}}, {"1261": {"logprob": -0.2108541578054428, "rank": 1, "decoded_token": " a"}, "32656": {"logprob": -1.710854172706604, "rank": 2, "decoded_token": " wooden"}, "17253": {"logprob": -5.5858540534973145, "rank": 3, "decoded_token": " weather"}, "44130": {"logprob": -6.0858540534973145, "rank": 4, "decoded_token": " rust"}, "12603": {"logprob": -6.9608540534973145, "rank": 5, "decoded_token": " wood"}}, {"32656": {"logprob": -0.08556432276964188, "rank": 1, "decoded_token": " wooden"}, "44130": {"logprob": -2.710564374923706, "rank": 2, "decoded_token": " rust"}, "17253": {"logprob": -4.710564136505127, "rank": 3, "decoded_token": " weather"}, "12603": {"logprob": -5.960564136505127, "rank": 4, "decoded_token": " wood"}, "3403": {"logprob": -5.960564136505127, "rank": 5, "decoded_token": " text"}}, {"4691": {"logprob": -0.7751782536506653, "rank": 1, "decoded_token": " surface"}, "11237": {"logprob": -0.7751782536506653, "rank": 2, "decoded_token": " floor"}, "7042": {"logprob": -2.9001781940460205, "rank": 3, "decoded_token": " background"}, "28984": {"logprob": -4.1501784324646, "rank": 4, "decoded_token": " deck"}, "92504": {"logprob": -6.1501784324646, "rank": 5, "decoded_token": " backdrop"}}, {"1626": {"logprob": -0.12918435037136078, "rank": 1, "decoded_token": ".\n"}, "1044": {"logprob": -2.3791842460632324, "rank": 2, "decoded_token": ","}, "1046": {"logprob": -4.129184246063232, "rank": 3, "decoded_token": "."}, "1338": {"logprob": -5.129184246063232, "rank": 4, "decoded_token": ".\n\n"}, "7283": {"logprob": -5.629184246063232, "rank": 5, "decoded_token": " looking"}}, {"1050": {"logprob": -0.00017474555352237076, "rank": 1, "decoded_token": "2"}, "1256": {"logprob": -9.000174522399902, "rank": 2, "decoded_token": " "}, "1032": {"logprob": -10.875174522399902, "rank": 3, "decoded_token": " "}, "1293": {"logprob": -11.625174522399902, "rank": 4, "decoded_token": " "}, "1051": {"logprob": -12.125174522399902, "rank": 5, "decoded_token": "3"}}, {"1046": {"logprob": -7.629365427419543e-06, "rank": 1, "decoded_token": "."}, "3590": {"logprob": -12.875007629394531, "rank": 2, "decoded_token": ".A"}, "1626": {"logprob": -13.062507629394531, "rank": 3, "decoded_token": ".\n"}, "1338": {"logprob": -14.562507629394531, "rank": 4, "decoded_token": ".\n\n"}, "1058": {"logprob": -14.812507629394531, "rank": 5, "decoded_token": ":"}}, {"1349": {"logprob": -0.558266282081604, "rank": 1, "decoded_token": " A"}, "11826": {"logprob": -1.495766282081604, "rank": 2, "decoded_token": " Maj"}, "37159": {"logprob": -2.2457661628723145, "rank": 3, "decoded_token": " Snow"}, "113465": {"logprob": -3.9957661628723145, "rank": 4, "decoded_token": " Rug"}, "1531": {"logprob": -3.9957661628723145, "rank": 5, "decoded_token": " The"}}, {"15375": {"logprob": -0.6446555852890015, "rank": 1, "decoded_token": " vast"}, "37849": {"logprob": -2.019655704498291, "rank": 2, "decoded_token": " breat"}, "61082": {"logprob": -2.394655704498291, "rank": 3, "decoded_token": " panor"}, "10726": {"logprob": -3.082155704498291, "rank": 4, "decoded_token": " scen"}, "2169": {"logprob": -3.207155704498291, "rank": 5, "decoded_token": " ser"}}, {"24361": {"logprob": -0.7034653425216675, "rank": 1, "decoded_token": " mountain"}, "127945": {"logprob": -1.9534653425216675, "rank": 2, "decoded_token": " mountainous"}, "1044": {"logprob": -2.078465461730957, "rank": 3, "decoded_token": ","}, "4521": {"logprob": -2.328465461730957, "rank": 4, "decoded_token": " range"}, "28035": {"logprob": -2.453465461730957, "rank": 5, "decoded_token": " landscape"}}, {"4521": {"logprob": -0.07058106362819672, "rank": 1, "decoded_token": " range"}, "28035": {"logprob": -2.6955809593200684, "rank": 2, "decoded_token": " landscape"}, "37691": {"logprob": -8.320581436157227, "rank": 3, "decoded_token": " valley"}, "12248": {"logprob": -9.445581436157227, "rank": 4, "decoded_token": " peak"}, "13327": {"logprob": -9.695581436157227, "rank": 5, "decoded_token": " scene"}}, {"1454": {"logprob": -1.1448894739151, "rank": 1, "decoded_token": " with"}, "94973": {"logprob": -1.1448894739151, "rank": 2, "decoded_token": " stretches"}, "2425": {"logprob": -1.8948894739151, "rank": 3, "decoded_token": " under"}, "1395": {"logprob": -2.5198893547058105, "rank": 4, "decoded_token": " is"}, "13875": {"logprob": -3.0198893547058105, "rank": 5, "decoded_token": " covered"}}, {"122203": {"logprob": -1.0288245677947998, "rank": 1, "decoded_token": " rugged"}, "58127": {"logprob": -1.6538245677947998, "rank": 2, "decoded_token": " jag"}, "27469": {"logprob": -2.1538245677948, "rank": 3, "decoded_token": " peaks"}, "23745": {"logprob": -2.6538245677948, "rank": 4, "decoded_token": " snow"}, "95746": {"logprob": -2.8413245677948, "rank": 5, "decoded_token": " rocky"}}, {"27469": {"logprob": -0.20564845204353333, "rank": 1, "decoded_token": " peaks"}, "24765": {"logprob": -2.580648422241211, "rank": 2, "decoded_token": " terrain"}, "130655": {"logprob": -2.955648422241211, "rank": 3, "decoded_token": ""}, "1044": {"logprob": -3.580648422241211, "rank": 4, "decoded_token": ","}, "61263": {"logprob": -4.455648422241211, "rank": 5, "decoded_token": " slopes"}}, {"94973": {"logprob": -1.0839273929595947, "rank": 1, "decoded_token": " stretches"}, "1321": {"logprob": -1.1464273929595947, "rank": 2, "decoded_token": " and"}, "2425": {"logprob": -1.7714273929595947, "rank": 3, "decoded_token": " under"}, "13875": {"logprob": -3.0839273929595947, "rank": 4, "decoded_token": " covered"}, "1395": {"logprob": -3.2714273929595947, "rank": 5, "decoded_token": " is"}}, {"2425": {"logprob": -0.9016233682632446, "rank": 1, "decoded_token": " under"}, "5669": {"logprob": -1.0266233682632446, "rank": 2, "decoded_token": " across"}, "1848": {"logprob": -1.9016233682632446, "rank": 3, "decoded_token": " out"}, "2203": {"logprob": -3.151623249053955, "rank": 4, "decoded_token": " into"}, "8994": {"logprob": -4.026623249053955, "rank": 5, "decoded_token": " towards"}}, {"1261": {"logprob": -0.00555459875613451, "rank": 1, "decoded_token": " a"}, "1420": {"logprob": -5.380554676055908, "rank": 2, "decoded_token": " an"}, "1278": {"logprob": -7.630554676055908, "rank": 3, "decoded_token": " the"}, "2136": {"logprob": -9.31805419921875, "rank": 4, "decoded_token": " over"}, "16152": {"logprob": -9.38055419921875, "rank": 5, "decoded_token": " cloud"}}, {"16152": {"logprob": -0.6862213015556335, "rank": 1, "decoded_token": " cloud"}, "6133": {"logprob": -1.4362213611602783, "rank": 2, "decoded_token": " clear"}, "18416": {"logprob": -2.6862213611602783, "rank": 3, "decoded_token": " haz"}, "27254": {"logprob": -3.0612213611602783, "rank": 4, "decoded_token": " partly"}, "4391": {"logprob": -3.1862213611602783, "rank": 5, "decoded_token": " light"}}, {"1121": {"logprob": -0.10446903109550476, "rank": 1, "decoded_token": "y"}, "4527": {"logprob": -2.854469060897827, "rank": 2, "decoded_token": "less"}, "1286": {"logprob": -3.479469060897827, "rank": 3, "decoded_token": "ed"}, "114525": {"logprob": -5.479468822479248, "rank": 4, "decoded_token": "-covered"}, "77187": {"logprob": -5.479468822479248, "rank": 5, "decoded_token": "-filled"}}, {"21283": {"logprob": -0.003459066851064563, "rank": 1, "decoded_token": " sky"}, "10991": {"logprob": -6.3784589767456055, "rank": 2, "decoded_token": " blue"}, "1044": {"logprob": -6.8784589767456055, "rank": 3, "decoded_token": ","}, "26549": {"logprob": -7.8784589767456055, "rank": 4, "decoded_token": " gray"}, "34052": {"logprob": -8.503458976745605, "rank": 5, "decoded_token": " grey"}}, {"1046": {"logprob": -0.01103890035301447, "rank": 1, "decoded_token": "."}, "1044": {"logprob": -4.636038780212402, "rank": 2, "decoded_token": ","}, "1338": {"logprob": -7.261038780212402, "rank": 3, "decoded_token": ".\n\n"}, "1294": {"logprob": -8.136038780212402, "rank": 4, "decoded_token": " in"}, "1454": {"logprob": -8.761038780212402, "rank": 5, "decoded_token": " with"}}, {"2": {"logprob": -9.059865078597795e-06, "rank": 1, "decoded_token": ""}, "1032": {"logprob": -11.625008583068848, "rank": 2, "decoded_token": " "}, "1256": {"logprob": -16.125009536743164, "rank": 3, "decoded_token": " "}, "1319": {"logprob": -17.375009536743164, "rank": 4, "decoded_token": " ("}, "1766": {"logprob": -18.750009536743164, "rank": 5, "decoded_token": " ["}}]], [[1049, 1046, 1349, 7244, 10575, 53048, 41132, 3923, 1408, 1261, 32656, 11237, 1626, 1050, 1046, 1349, 15375, 24361, 4521, 94973, 5669, 1278, 48932, 2425, 1261, 16152, 1121, 21283, 1626, 1051, 1046, 8342, 71284, 7377, 1394, 22140, 1294, 1278, 27208, 1513, 97558, 1626, 1052, 1046, 1349, 53301, 59396, 3549, 13335, 2645, 1261, 1295, 3506, 11223, 12097, 1046, 2], "1. A black dog sits attentively on a wooden floor.\n2. A vast mountain range stretches across the horizon under a cloudy sky.\n3. Surfers wait for waves in the ocean at sunset.\n4. A winding gravel path leads through a lush green park.", [{"1049": {"logprob": -0.05001257359981537, "rank": 1, "decoded_token": "1"}, "1045": {"logprob": -3.1750125885009766, "rank": 2, "decoded_token": "-"}, "69957": {"logprob": -5.925012588500977, "rank": 3, "decoded_token": "Sure"}, "11745": {"logprob": -6.425012588500977, "rank": 4, "decoded_token": "Here"}, "1065": {"logprob": -6.425012588500977, "rank": 5, "decoded_token": "A"}}, {"1046": {"logprob": -9.536697689327411e-06, "rank": 1, "decoded_token": "."}, "1058": {"logprob": -11.875009536743164, "rank": 2, "decoded_token": ":"}, "3590": {"logprob": -13.375009536743164, "rank": 3, "decoded_token": ".A"}, "1041": {"logprob": -14.750009536743164, "rank": 4, "decoded_token": ")"}, "1065": {"logprob": -15.687509536743164, "rank": 5, "decoded_token": "A"}}, {"1349": {"logprob": -0.12580634653568268, "rank": 1, "decoded_token": " A"}, "1429": {"logprob": -2.3758063316345215, "rank": 2, "decoded_token": " \""}, "1531": {"logprob": -4.6258063316345215, "rank": 3, "decoded_token": " The"}, "11967": {"logprob": -4.6258063316345215, "rank": 4, "decoded_token": " Image"}, "1603": {"logprob": -5.6258063316345215, "rank": 5, "decoded_token": " **"}}, {"7244": {"logprob": -0.15412142872810364, "rank": 1, "decoded_token": " black"}, "68076": {"logprob": -3.3416213989257812, "rank": 2, "decoded_token": " cute"}, "6231": {"logprob": -3.9666213989257812, "rank": 3, "decoded_token": " close"}, "38462": {"logprob": -4.216621398925781, "rank": 4, "decoded_token": " curious"}, "4329": {"logprob": -4.404121398925781, "rank": 5, "decoded_token": " large"}}, {"10575": {"logprob": -0.12086891382932663, "rank": 1, "decoded_token": " dog"}, "116572": {"logprob": -2.3708689212799072, "rank": 2, "decoded_token": " puppy"}, "119075": {"logprob": -3.9958689212799072, "rank": 3, "decoded_token": " Labrador"}, "15812": {"logprob": -7.683368682861328, "rank": 4, "decoded_token": " Lab"}, "8636": {"logprob": -7.808368682861328, "rank": 5, "decoded_token": " lab"}}, {"53048": {"logprob": -0.8729249238967896, "rank": 1, "decoded_token": " sits"}, "1454": {"logprob": -1.1229249238967896, "rank": 2, "decoded_token": " with"}, "1395": {"logprob": -2.4354248046875, "rank": 3, "decoded_token": " is"}, "18970": {"logprob": -2.6854248046875, "rank": 4, "decoded_token": " sitting"}, "22524": {"logprob": -3.6854248046875, "rank": 5, "decoded_token": " lies"}}, {"41132": {"logprob": -0.5888903737068176, "rank": 1, "decoded_token": " attent"}, "106534": {"logprob": -1.2763903141021729, "rank": 2, "decoded_token": " calmly"}, "12276": {"logprob": -2.838890314102173, "rank": 3, "decoded_token": " alert"}, "1408": {"logprob": -2.901390314102173, "rank": 4, "decoded_token": " on"}, "6482": {"logprob": -5.026390552520752, "rank": 5, "decoded_token": " patient"}}, {"3923": {"logprob": -9.16677454370074e-05, "rank": 1, "decoded_token": "ively"}, "1556": {"logprob": -9.625091552734375, "rank": 2, "decoded_token": "ive"}, "6655": {"logprob": -10.875091552734375, "rank": 3, "decoded_token": "atively"}, "3929": {"logprob": -13.125091552734375, "rank": 4, "decoded_token": "ently"}, "47885": {"logprob": -13.750091552734375, "rank": 5, "decoded_token": "edly"}}, {"1408": {"logprob": -0.052677519619464874, "rank": 1, "decoded_token": " on"}, "3675": {"logprob": -3.802677631378174, "rank": 2, "decoded_token": " against"}, "1454": {"logprob": -4.302677631378174, "rank": 3, "decoded_token": " with"}, "1294": {"logprob": -5.177677631378174, "rank": 4, "decoded_token": " in"}, "7283": {"logprob": -5.427677631378174, "rank": 5, "decoded_token": " looking"}}, {"1261": {"logprob": -0.36706605553627014, "rank": 1, "decoded_token": " a"}, "32656": {"logprob": -1.2420660257339478, "rank": 2, "decoded_token": " wooden"}, "17253": {"logprob": -4.617065906524658, "rank": 3, "decoded_token": " weather"}, "44130": {"logprob": -5.742065906524658, "rank": 4, "decoded_token": " rust"}, "12603": {"logprob": -6.617065906524658, "rank": 5, "decoded_token": " wood"}}, {"32656": {"logprob": -0.07824385166168213, "rank": 1, "decoded_token": " wooden"}, "44130": {"logprob": -2.8282437324523926, "rank": 2, "decoded_token": " rust"}, "17253": {"logprob": -4.703243732452393, "rank": 3, "decoded_token": " weather"}, "12603": {"logprob": -5.828243732452393, "rank": 4, "decoded_token": " wood"}, "3403": {"logprob": -5.953243732452393, "rank": 5, "decoded_token": " text"}}, {"11237": {"logprob": -0.5853750705718994, "rank": 1, "decoded_token": " floor"}, "4691": {"logprob": -1.0853750705718994, "rank": 2, "decoded_token": " surface"}, "7042": {"logprob": -2.7103750705718994, "rank": 3, "decoded_token": " background"}, "28984": {"logprob": -3.5853750705718994, "rank": 4, "decoded_token": " deck"}, "92504": {"logprob": -6.08537483215332, "rank": 5, "decoded_token": " backdrop"}}, {"1626": {"logprob": -0.7340722680091858, "rank": 1, "decoded_token": ".\n"}, "1044": {"logprob": -0.8590722680091858, "rank": 2, "decoded_token": ","}, "1454": {"logprob": -3.359072208404541, "rank": 3, "decoded_token": " with"}, "7283": {"logprob": -3.609072208404541, "rank": 4, "decoded_token": " looking"}, "1321": {"logprob": -4.109072208404541, "rank": 5, "decoded_token": " and"}}, {"1050": {"logprob": -1.1324817933200393e-05, "rank": 1, "decoded_token": "2"}, "1051": {"logprob": -11.625011444091797, "rank": 2, "decoded_token": "3"}, "1256": {"logprob": -14.000011444091797, "rank": 3, "decoded_token": " "}, "1049": {"logprob": -14.625011444091797, "rank": 4, "decoded_token": "1"}, "1032": {"logprob": -14.625011444091797, "rank": 5, "decoded_token": " "}}, {"1046": {"logprob": -2.50339189733495e-06, "rank": 1, "decoded_token": "."}, "3590": {"logprob": -13.56250286102295, "rank": 2, "decoded_token": ".A"}, "1626": {"logprob": -15.43750286102295, "rank": 3, "decoded_token": ".\n"}, "4700": {"logprob": -15.50000286102295, "rank": 4, "decoded_token": ".M"}, "3051": {"logprob": -16.000001907348633, "rank": 5, "decoded_token": ".S"}}, {"1349": {"logprob": -0.6769706010818481, "rank": 1, "decoded_token": " A"}, "11826": {"logprob": -1.9269706010818481, "rank": 2, "decoded_token": " Maj"}, "37159": {"logprob": -2.1144704818725586, "rank": 3, "decoded_token": " Snow"}, "27260": {"logprob": -2.6144704818725586, "rank": 4, "decoded_token": " Mountain"}, "113465": {"logprob": -2.8644704818725586, "rank": 5, "decoded_token": " Rug"}}, {"15375": {"logprob": -0.9251430034637451, "rank": 1, "decoded_token": " vast"}, "10726": {"logprob": -2.300143003463745, "rank": 2, "decoded_token": " scen"}, "4521": {"logprob": -2.362643003463745, "rank": 3, "decoded_token": " range"}, "122203": {"logprob": -2.425143003463745, "rank": 4, "decoded_token": " rugged"}, "61082": {"logprob": -2.800143003463745, "rank": 5, "decoded_token": " panor"}}, {"24361": {"logprob": -0.5277582406997681, "rank": 1, "decoded_token": " mountain"}, "127945": {"logprob": -1.902758240699768, "rank": 2, "decoded_token": " mountainous"}, "28035": {"logprob": -2.5277581214904785, "rank": 3, "decoded_token": " landscape"}, "4521": {"logprob": -2.5277581214904785, "rank": 4, "decoded_token": " range"}, "1044": {"logprob": -2.7777581214904785, "rank": 5, "decoded_token": ","}}, {"4521": {"logprob": -0.055658817291259766, "rank": 1, "decoded_token": " range"}, "28035": {"logprob": -2.9306588172912598, "rank": 2, "decoded_token": " landscape"}, "37691": {"logprob": -8.430658340454102, "rank": 3, "decoded_token": " valley"}, "13327": {"logprob": -9.055658340454102, "rank": 4, "decoded_token": " scene"}, "3719": {"logprob": -9.805658340454102, "rank": 5, "decoded_token": " view"}}, {"94973": {"logprob": -0.6880245208740234, "rank": 1, "decoded_token": " stretches"}, "2425": {"logprob": -1.7505245208740234, "rank": 2, "decoded_token": " under"}, "1395": {"logprob": -2.3130245208740234, "rank": 3, "decoded_token": " is"}, "1454": {"logprob": -2.6880245208740234, "rank": 4, "decoded_token": " with"}, "7038": {"logprob": -3.2505245208740234, "rank": 5, "decoded_token": " extends"}}, {"5669": {"logprob": -0.4545598328113556, "rank": 1, "decoded_token": " across"}, "2425": {"logprob": -1.4545598030090332, "rank": 2, "decoded_token": " under"}, "1848": {"logprob": -2.454559803009033, "rank": 3, "decoded_token": " out"}, "2203": {"logprob": -4.204559803009033, "rank": 4, "decoded_token": " into"}, "25136": {"logprob": -4.642059803009033, "rank": 5, "decoded_token": " beneath"}}, {"1278": {"logprob": -0.23015151917934418, "rank": 1, "decoded_token": " the"}, "1261": {"logprob": -1.6051515340805054, "rank": 2, "decoded_token": " a"}, "1420": {"logprob": -5.605151653289795, "rank": 3, "decoded_token": " an"}, "2425": {"logprob": -7.167651653289795, "rank": 4, "decoded_token": " under"}, "1454": {"logprob": -10.167651176452637, "rank": 5, "decoded_token": " with"}}, {"48932": {"logprob": -0.2797861397266388, "rank": 1, "decoded_token": " horizon"}, "21283": {"logprob": -2.0297861099243164, "rank": 2, "decoded_token": " sky"}, "3937": {"logprob": -3.2797861099243164, "rank": 3, "decoded_token": " image"}, "28035": {"logprob": -3.6547861099243164, "rank": 4, "decoded_token": " landscape"}, "3044": {"logprob": -3.7797861099243164, "rank": 5, "decoded_token": " sk"}}, {"2425": {"logprob": -0.28862035274505615, "rank": 1, "decoded_token": " under"}, "1044": {"logprob": -2.4136204719543457, "rank": 2, "decoded_token": ","}, "1454": {"logprob": -2.5386204719543457, "rank": 3, "decoded_token": " with"}, "1626": {"logprob": -3.7886204719543457, "rank": 4, "decoded_token": ".\n"}, "1408": {"logprob": -3.9136204719543457, "rank": 5, "decoded_token": " on"}}, {"1261": {"logprob": -0.04524127021431923, "rank": 1, "decoded_token": " a"}, "16152": {"logprob": -4.045241355895996, "rank": 2, "decoded_token": " cloud"}, "1420": {"logprob": -4.045241355895996, "rank": 3, "decoded_token": " an"}, "2136": {"logprob": -6.107741355895996, "rank": 4, "decoded_token": " over"}, "6133": {"logprob": -6.357741355895996, "rank": 5, "decoded_token": " clear"}}, {"16152": {"logprob": -0.19613930583000183, "rank": 1, "decoded_token": " cloud"}, "6133": {"logprob": -2.883639335632324, "rank": 2, "decoded_token": " clear"}, "27254": {"logprob": -3.508639335632324, "rank": 3, "decoded_token": " partly"}, "18416": {"logprob": -3.883639335632324, "rank": 4, "decoded_token": " haz"}, "4391": {"logprob": -4.321139335632324, "rank": 5, "decoded_token": " light"}}, {"1121": {"logprob": -0.05146069824695587, "rank": 1, "decoded_token": "y"}, "1286": {"logprob": -3.8014607429504395, "rank": 2, "decoded_token": "ed"}, "77187": {"logprob": -4.5514607429504395, "rank": 3, "decoded_token": "-filled"}, "114525": {"logprob": -4.9264607429504395, "rank": 4, "decoded_token": "-covered"}, "4527": {"logprob": -4.9264607429504395, "rank": 5, "decoded_token": "less"}}, {"21283": {"logprob": -0.00033122775494121015, "rank": 1, "decoded_token": " sky"}, "10991": {"logprob": -8.875330924987793, "rank": 2, "decoded_token": " blue"}, "1044": {"logprob": -9.500330924987793, "rank": 3, "decoded_token": ","}, "26549": {"logprob": -10.500330924987793, "rank": 4, "decoded_token": " gray"}, "34052": {"logprob": -11.375330924987793, "rank": 5, "decoded_token": " grey"}}, {"1626": {"logprob": -0.00012683063687290996, "rank": 1, "decoded_token": ".\n"}, "1044": {"logprob": -9.500126838684082, "rank": 2, "decoded_token": ","}, "1046": {"logprob": -10.500126838684082, "rank": 3, "decoded_token": "."}, "1454": {"logprob": -10.875126838684082, "rank": 4, "decoded_token": " with"}, "1294": {"logprob": -13.375126838684082, "rank": 5, "decoded_token": " in"}}, {"1051": {"logprob": -3.2186455882765586e-06, "rank": 1, "decoded_token": "3"}, "1052": {"logprob": -12.75000286102295, "rank": 2, "decoded_token": "4"}, "1050": {"logprob": -15.00000286102295, "rank": 3, "decoded_token": "2"}, "1049": {"logprob": -17.000003814697266, "rank": 4, "decoded_token": "1"}, "1032": {"logprob": -17.937503814697266, "rank": 5, "decoded_token": " "}}, {"1046": {"logprob": -1.9073468138230965e-06, "rank": 1, "decoded_token": "."}, "3590": {"logprob": -14.625001907348633, "rank": 2, "decoded_token": ".A"}, "5226": {"logprob": -15.625001907348633, "rank": 3, "decoded_token": ".D"}, "6847": {"logprob": -15.750001907348633, "rank": 4, "decoded_token": ".T"}, "4700": {"logprob": -16.750001907348633, "rank": 5, "decoded_token": ".M"}}, {"8342": {"logprob": -0.5928499102592468, "rank": 1, "decoded_token": " Sur"}, "1349": {"logprob": -1.6553499698638916, "rank": 2, "decoded_token": " A"}, "22468": {"logprob": -2.5303499698638916, "rank": 3, "decoded_token": " Several"}, "1488": {"logprob": -2.7178499698638916, "rank": 4, "decoded_token": " W"}, "15035": {"logprob": -3.2178499698638916, "rank": 5, "decoded_token": " People"}}, {"71284": {"logprob": -0.003268140833824873, "rank": 1, "decoded_token": "fers"}, "1102": {"logprob": -5.878268241882324, "rank": 2, "decoded_token": "f"}, "1726": {"logprob": -7.753268241882324, "rank": 3, "decoded_token": "fer"}, "61888": {"logprob": -12.315768241882324, "rank": 4, "decoded_token": "fline"}, "2119": {"logprob": -13.065768241882324, "rank": 5, "decoded_token": "fter"}}, {"7377": {"logprob": -1.4883846044540405, "rank": 1, "decoded_token": " wait"}, "1584": {"logprob": -1.7383846044540405, "rank": 2, "decoded_token": " are"}, "88014": {"logprob": -1.9258846044540405, "rank": 3, "decoded_token": " paddle"}, "1294": {"logprob": -1.9258846044540405, "rank": 4, "decoded_token": " in"}, "24434": {"logprob": -2.23838472366333, "rank": 5, "decoded_token": " ride"}}, {"1394": {"logprob": -0.6120346188545227, "rank": 1, "decoded_token": " for"}, "1294": {"logprob": -0.9870346188545227, "rank": 2, "decoded_token": " in"}, "1408": {"logprob": -2.737034559249878, "rank": 3, "decoded_token": " on"}, "6482": {"logprob": -4.487034797668457, "rank": 4, "decoded_token": " patient"}, "1321": {"logprob": -5.612034797668457, "rank": 5, "decoded_token": " and"}}, {"22140": {"logprob": -0.008224429562687874, "rank": 1, "decoded_token": " waves"}, "1278": {"logprob": -5.5082244873046875, "rank": 2, "decoded_token": " the"}, "1261": {"logprob": -5.6332244873046875, "rank": 3, "decoded_token": " a"}, "39460": {"logprob": -8.133224487304688, "rank": 4, "decoded_token": " incoming"}, "1321": {"logprob": -9.758224487304688, "rank": 5, "decoded_token": " and"}}, {"1294": {"logprob": -0.3204176723957062, "rank": 1, "decoded_token": " in"}, "1408": {"logprob": -2.195417642593384, "rank": 2, "decoded_token": " on"}, "1513": {"logprob": -2.320417642593384, "rank": 3, "decoded_token": " at"}, "3016": {"logprob": -3.695417642593384, "rank": 4, "decoded_token": " while"}, "1435": {"logprob": -3.820417642593384, "rank": 5, "decoded_token": " as"}}, {"1278": {"logprob": -0.004615250043570995, "rank": 1, "decoded_token": " the"}, "1261": {"logprob": -6.192115306854248, "rank": 2, "decoded_token": " a"}, "1420": {"logprob": -6.942115306854248, "rank": 3, "decoded_token": " an"}, "40466": {"logprob": -7.317115306854248, "rank": 4, "decoded_token": " shallow"}, "26517": {"logprob": -7.879615306854248, "rank": 5, "decoded_token": " calm"}}, {"27208": {"logprob": -0.06491076946258545, "rank": 1, "decoded_token": " ocean"}, "7786": {"logprob": -3.439910888671875, "rank": 2, "decoded_token": " distance"}, "5124": {"logprob": -5.314910888671875, "rank": 3, "decoded_token": " early"}, "26517": {"logprob": -5.377410888671875, "rank": 4, "decoded_token": " calm"}, "11196": {"logprob": -5.377410888671875, "rank": 5, "decoded_token": " sea"}}, {"1513": {"logprob": -1.144903540611267, "rank": 1, "decoded_token": " at"}, "1435": {"logprob": -1.269903540611267, "rank": 2, "decoded_token": " as"}, "3184": {"logprob": -1.394903540611267, "rank": 3, "decoded_token": " during"}, "3016": {"logprob": -3.0199036598205566, "rank": 4, "decoded_token": " while"}, "6117": {"logprob": -3.1449036598205566, "rank": 5, "decoded_token": " near"}}, {"97558": {"logprob": -0.12556149065494537, "rank": 1, "decoded_token": " sunset"}, "11729": {"logprob": -2.875561475753784, "rank": 2, "decoded_token": " sun"}, "1266": {"logprob": -3.375561475753784, "rank": 3, "decoded_token": " d"}, "54507": {"logprob": -4.000561714172363, "rank": 4, "decoded_token": " dawn"}, "1261": {"logprob": -5.125561714172363, "rank": 5, "decoded_token": " a"}}, {"1626": {"logprob": -0.26737067103385925, "rank": 1, "decoded_token": ".\n"}, "1044": {"logprob": -2.2673707008361816, "rank": 2, "decoded_token": ","}, "3016": {"logprob": -2.7673707008361816, "rank": 3, "decoded_token": " while"}, "1454": {"logprob": -3.5173707008361816, "rank": 4, "decoded_token": " with"}, "6117": {"logprob": -4.142370700836182, "rank": 5, "decoded_token": " near"}}, {"1052": {"logprob": -2.9802276912960224e-06, "rank": 1, "decoded_token": "4"}, "1051": {"logprob": -13.37500286102295, "rank": 2, "decoded_token": "3"}, "1049": {"logprob": -14.00000286102295, "rank": 3, "decoded_token": "1"}, "1053": {"logprob": -14.56250286102295, "rank": 4, "decoded_token": "5"}, "1032": {"logprob": -16.750003814697266, "rank": 5, "decoded_token": " "}}, {"1046": {"logprob": -1.6689286894688848e-06, "rank": 1, "decoded_token": "."}, "3590": {"logprob": -13.500001907348633, "rank": 2, "decoded_token": ".A"}, "6847": {"logprob": -16.562501907348633, "rank": 3, "decoded_token": ".T"}, "1044": {"logprob": -17.312501907348633, "rank": 4, "decoded_token": ","}, "1349": {"logprob": -17.500001907348633, "rank": 5, "decoded_token": " A"}}, {"1349": {"logprob": -0.004883386194705963, "rank": 1, "decoded_token": " A"}, "2048": {"logprob": -5.504883289337158, "rank": 2, "decoded_token": " An"}, "10638": {"logprob": -7.754883289337158, "rank": 3, "decoded_token": " Two"}, "111463": {"logprob": -9.754883766174316, "rank": 4, "decoded_token": " Trees"}, "1531": {"logprob": -10.692383766174316, "rank": 5, "decoded_token": " The"}}, {"53301": {"logprob": -1.5612412691116333, "rank": 1, "decoded_token": " winding"}, "15192": {"logprob": -1.7487412691116333, "rank": 2, "decoded_token": " narrow"}, "47945": {"logprob": -2.1237411499023438, "rank": 3, "decoded_token": " dirt"}, "2169": {"logprob": -2.5612411499023438, "rank": 4, "decoded_token": " ser"}, "59396": {"logprob": -2.6862411499023438, "rank": 5, "decoded_token": " gravel"}}, {"59396": {"logprob": -0.9024254083633423, "rank": 1, "decoded_token": " gravel"}, "3549": {"logprob": -1.1524254083633423, "rank": 2, "decoded_token": " path"}, "47945": {"logprob": -1.6524254083633423, "rank": 3, "decoded_token": " dirt"}, "14801": {"logprob": -3.1524252891540527, "rank": 4, "decoded_token": " pathway"}, "15551": {"logprob": -4.277425289154053, "rank": 5, "decoded_token": " stone"}}, {"3549": {"logprob": -0.021290099248290062, "rank": 1, "decoded_token": " path"}, "14801": {"logprob": -3.8962900638580322, "rank": 2, "decoded_token": " pathway"}, "33659": {"logprob": -7.896290302276611, "rank": 3, "decoded_token": " trail"}, "9480": {"logprob": -9.521289825439453, "rank": 4, "decoded_token": " road"}, "7368": {"logprob": -9.646289825439453, "rank": 5, "decoded_token": "path"}}, {"13335": {"logprob": -0.16593234241008759, "rank": 1, "decoded_token": " leads"}, "39985": {"logprob": -2.8534324169158936, "rank": 2, "decoded_token": " cuts"}, "1639": {"logprob": -3.9784324169158936, "rank": 3, "decoded_token": " me"}, "11500": {"logprob": -4.1034321784973145, "rank": 4, "decoded_token": " runs"}, "2645": {"logprob": -4.2909321784973145, "rank": 5, "decoded_token": " through"}}, {"2645": {"logprob": -0.05767015367746353, "rank": 1, "decoded_token": " through"}, "8994": {"logprob": -4.0576701164245605, "rank": 2, "decoded_token": " towards"}, "2396": {"logprob": -4.1826701164245605, "rank": 3, "decoded_token": " between"}, "2203": {"logprob": -4.5576701164245605, "rank": 4, "decoded_token": " into"}, "1317": {"logprob": -5.5576701164245605, "rank": 5, "decoded_token": " to"}}, {"1261": {"logprob": -0.017209367826581, "rank": 1, "decoded_token": " a"}, "11223": {"logprob": -4.892209529876709, "rank": 2, "decoded_token": " green"}, "1295": {"logprob": -5.017209529876709, "rank": 3, "decoded_token": " l"}, "23170": {"logprob": -6.767209529876709, "rank": 4, "decoded_token": " grass"}, "1420": {"logprob": -7.267209529876709, "rank": 5, "decoded_token": " an"}}, {"1295": {"logprob": -0.9430665969848633, "rank": 1, "decoded_token": " l"}, "11223": {"logprob": -1.3180665969848633, "rank": 2, "decoded_token": " green"}, "23170": {"logprob": -1.9430665969848633, "rank": 3, "decoded_token": " grass"}, "12097": {"logprob": -2.4430665969848633, "rank": 4, "decoded_token": " park"}, "26428": {"logprob": -3.3180665969848633, "rank": 5, "decoded_token": " garden"}}, {"3506": {"logprob": -6.556489552167477e-06, "rank": 1, "decoded_token": "ush"}, "1374": {"logprob": -12.000006675720215, "rank": 2, "decoded_token": "us"}, "90716": {"logprob": -15.625006675720215, "rank": 3, "decoded_token": "USH"}, "16938": {"logprob": -15.875006675720215, "rank": 4, "decoded_token": "usher"}, "13326": {"logprob": -17.1875057220459, "rank": 5, "decoded_token": "inden"}}, {"11223": {"logprob": -0.36697858572006226, "rank": 1, "decoded_token": " green"}, "1044": {"logprob": -1.366978645324707, "rank": 2, "decoded_token": ","}, "26428": {"logprob": -3.491978645324707, "rank": 3, "decoded_token": " garden"}, "12097": {"logprob": -4.116978645324707, "rank": 4, "decoded_token": " park"}, "23170": {"logprob": -5.866978645324707, "rank": 5, "decoded_token": " grass"}}, {"12097": {"logprob": -0.5570574402809143, "rank": 1, "decoded_token": " park"}, "3727": {"logprob": -1.9320573806762695, "rank": 2, "decoded_token": " field"}, "28035": {"logprob": -2.1820573806762695, "rank": 3, "decoded_token": " landscape"}, "26428": {"logprob": -2.4320573806762695, "rank": 4, "decoded_token": " garden"}, "4457": {"logprob": -2.8070573806762695, "rank": 5, "decoded_token": " area"}}, {"1046": {"logprob": -0.7940837144851685, "rank": 1, "decoded_token": "."}, "1454": {"logprob": -1.2940837144851685, "rank": 2, "decoded_token": " with"}, "8994": {"logprob": -2.794083595275879, "rank": 3, "decoded_token": " towards"}, "54410": {"logprob": -3.544083595275879, "rank": 4, "decoded_token": " lined"}, "2425": {"logprob": -3.544083595275879, "rank": 5, "decoded_token": " under"}}, {"2": {"logprob": -2.145764938177308e-06, "rank": 1, "decoded_token": ""}, "1032": {"logprob": -13.125001907348633, "rank": 2, "decoded_token": " "}, "1256": {"logprob": -16.000001907348633, "rank": 3, "decoded_token": " "}, "1293": {"logprob": -18.750001907348633, "rank": 4, "decoded_token": " "}, "1319": {"logprob": -19.687501907348633, "rank": 5, "decoded_token": " ("}}]]]
\ No newline at end of file
diff --git a/asian_ai/performance/vLLM/tests/models/fixtures/qwen2_5_math_prm_reward_step.json b/asian_ai/performance/vLLM/tests/models/fixtures/qwen2_5_math_prm_reward_step.json
new file mode 100644
index 0000000000000000000000000000000000000000..dc0f3010cc3aa59fd77e9e09142ef74eb5228b6d
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/fixtures/qwen2_5_math_prm_reward_step.json
@@ -0,0 +1 @@
+[[[0.0006361007690429688, 0.99951171875], [0.81884765625, 0.1812744140625], [0.025543212890625, 0.974609375], [0.0004382133483886719, 0.99951171875]]]
\ No newline at end of file
diff --git a/asian_ai/performance/vLLM/tests/models/language/__init__.py b/asian_ai/performance/vLLM/tests/models/language/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation/__init__.py b/asian_ai/performance/vLLM/tests/models/language/generation/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation/conftest.py b/asian_ai/performance/vLLM/tests/models/language/generation/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..d5e032cab73c557be43839ab069e11b6f26a842a
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/generation/conftest.py
@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Pytest configuration for vLLM language generation tests."""
+
+import os
+import warnings
+
+import torch
+
+from vllm.platforms import current_platform
+
+
+def pytest_configure(config):
+ """Early ROCm configuration that must happen before test collection."""
+ if not current_platform.is_rocm():
+ return
+
+ # Disable skinny GEMM on ROCm to avoid non-deterministic results
+ # from atomic reductions in wvSplitKrc kernel.
+ # See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975
+ os.environ["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
+ warnings.warn(
+ "ROCm: Set VLLM_ROCM_USE_SKINNY_GEMM=0 to avoid non-deterministic "
+ "results from skinny GEMM atomic reductions",
+ UserWarning,
+ stacklevel=1,
+ )
+
+
+def pytest_sessionstart(session):
+ """Configure ROCm-specific settings before test session starts."""
+ if not current_platform.is_rocm():
+ return
+
+ # Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers
+ # accuracy issues: https://github.com/vllm-project/vllm/issues/30167
+ # TODO: Remove once ROCm SDP accuracy issues are resolved on HuggingFace
+ torch.backends.cuda.enable_flash_sdp(False)
+ torch.backends.cuda.enable_mem_efficient_sdp(False)
+ torch.backends.cuda.enable_math_sdp(True)
+ warnings.warn(
+ "ROCm: Disabled flash_sdp and mem_efficient_sdp, enabled math_sdp "
+ "to avoid HuggingFace Transformers accuracy issues",
+ UserWarning,
+ stacklevel=1,
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation/test_common.py b/asian_ai/performance/vLLM/tests/models/language/generation/test_common.py
new file mode 100644
index 0000000000000000000000000000000000000000..8d5de51334da749cb0e8a20d07807ebdf08fdc52
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/generation/test_common.py
@@ -0,0 +1,232 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+import torch
+from packaging.version import Version
+from transformers import __version__ as TRANSFORMERS_VERSION
+
+from vllm.platforms import current_platform
+
+from ....utils import large_gpu_mark
+from ...registry import HF_EXAMPLE_MODELS
+from ...utils import check_logprobs_close
+
+# Models that require embedding scaling for prompt_embeds test
+EMBED_SCALING_MODELS = {
+ "openbmb/MiniCPM4.1-8B",
+}
+
+# This list contains the model that are using AITER kernel.
+# Skip model that are not using AITER tests.
+# When more AITER kernels are added, this list will not be
+# needed as all the models will be calling AITER kernels
+# in parts of the operators
+AITER_MODEL_LIST = [
+ "meta-llama/Llama-3.2-1B-Instruct",
+ "openbmb/MiniCPM3-4B",
+ "Qwen/Qwen2.5-0.5B-Instruct",
+ "TitanML/tiny-mixtral",
+ "Qwen/Qwen3-8B",
+]
+
+
+# @maybe_test_rocm_aiter
+@pytest.mark.parametrize(
+ "model",
+ [
+ pytest.param(
+ "bigscience/bloom-560m", # bloom - testing alibi slopes
+ marks=[
+ pytest.mark.core_model,
+ pytest.mark.slow_test,
+ pytest.mark.cpu_model,
+ ],
+ ),
+ pytest.param(
+ "openai-community/gpt2", # gpt2
+ marks=[pytest.mark.core_model],
+ ),
+ pytest.param("Milos/slovak-gpt-j-405M"), # gptj
+ pytest.param("bigcode/tiny_starcoder_py"), # gpt_bigcode
+ pytest.param("EleutherAI/pythia-70m"), # gpt_neox
+ pytest.param(
+ "google/gemma-1.1-2b-it", # gemma
+ marks=[
+ pytest.mark.core_model,
+ pytest.mark.cpu_model,
+ pytest.mark.slow_test,
+ ],
+ ),
+ pytest.param(
+ "google/gemma-2-2b-it", # test hybrid attention
+ marks=[pytest.mark.cpu_model],
+ ),
+ pytest.param(
+ "zai-org/chatglm3-6b", # chatglm (text-only)
+ ),
+ pytest.param(
+ "meta-llama/Llama-3.2-1B-Instruct", # llama
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
+ ),
+ pytest.param(
+ "openbmb/MiniCPM4.1-8B", # minicpm
+ marks=[pytest.mark.core_model, large_gpu_mark(min_gb=48)],
+ ),
+ pytest.param(
+ "facebook/opt-125m", # opt
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
+ ),
+ pytest.param(
+ "microsoft/phi-2", # phi
+ marks=[pytest.mark.core_model, pytest.mark.slow_test],
+ ),
+ pytest.param(
+ "Qwen/Qwen2.5-0.5B-Instruct", # qwen2
+ marks=[
+ pytest.mark.core_model,
+ pytest.mark.cpu_model,
+ pytest.mark.slow_test,
+ ],
+ ),
+ pytest.param(
+ "Qwen/Qwen3-8B", # qwen (text-only)
+ ),
+ pytest.param("stabilityai/stablelm-3b-4e1t"), # stablelm
+ pytest.param("bigcode/starcoder2-3b"), # starcoder2
+ pytest.param(
+ "TitanML/tiny-mixtral", # mixtral
+ marks=[pytest.mark.core_model],
+ ),
+ pytest.param("swiss-ai/Apertus-8B-Instruct-2509"), # apertus
+ pytest.param(
+ "naver-hyperclovax/HyperCLOVAX-SEED-Think-14B", # hyperclovax
+ marks=[large_gpu_mark(min_gb=32)],
+ ),
+ ],
+)
+@pytest.mark.parametrize("max_tokens", [32])
+@pytest.mark.parametrize("num_logprobs", [5])
+@pytest.mark.parametrize(
+ "use_rocm_aiter", [True, False] if current_platform.is_rocm() else [False]
+)
+@pytest.mark.parametrize("use_prompt_embeds", [True, False])
+def test_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ max_tokens: int,
+ num_logprobs: int,
+ use_rocm_aiter: bool,
+ use_prompt_embeds: bool,
+ monkeypatch,
+) -> None:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+
+ if use_rocm_aiter and (model in AITER_MODEL_LIST):
+ monkeypatch.setenv("VLLM_ROCM_USE_AITER", "1")
+ if model == "TitanML/tiny-mixtral":
+ # Untrained model: near-uniform logits make argmax sensitive to
+ # AITER's bfloat16 rounding error. Route the plain rms_norm and the
+ # fused MoE (whose near-uniform router logits flip expert selection
+ # under ~1 ULP drift) through the native kernels for this model.
+ # See ROCm/aiter#3806 for the tracking issue and minimal repro.
+ monkeypatch.setenv("VLLM_ROCM_USE_AITER_RMSNORM", "0")
+ monkeypatch.setenv("VLLM_ROCM_USE_AITER_MOE", "0")
+ elif use_rocm_aiter and model not in AITER_MODEL_LIST:
+ # Skip model that are not using AITER tests.
+ # When more AITER kernels are added, this list will not be
+ # needed as all the models will be calling AITER kernels
+ # in parts of the operators
+ pytest.skip(f"Skipping '{model}' model test with AITER kernel.")
+
+ if model == "bigcode/starcoder2-3b":
+ # Replace example.txt's Test1 (an NL prompt) with a code prompt:
+ # starcoder2-3b is a code model, so NL prompts give near-uniform
+ # digit logits where HF<->vLLM bf16 drift can reorder top-K.
+ example_prompts = list(example_prompts)
+ example_prompts[1] = (
+ "def add(a, b):\n return a + b\n\ndef sub(a, b):\n return a - "
+ )
+
+ with hf_runner(
+ model,
+ revision=model_info.revision,
+ trust_remote_code=model_info.trust_remote_code,
+ ) as hf_model:
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ prompt_embeds: list[torch.Tensor] | None = [] if use_prompt_embeds else None
+
+ for prompt in example_prompts:
+ token_ids = hf_model.tokenizer(prompt, return_tensors="pt").input_ids.to(
+ hf_model.model.device
+ )
+ if prompt_embeds is not None:
+ embed = hf_model.model.get_input_embeddings()(token_ids)
+
+ if "gemma" in model.lower() and (
+ Version(TRANSFORMERS_VERSION) < Version("5.3.0.dev0")
+ ):
+ # For Gemma 1/2 models with Transformers 5.4.0+, the prompt
+ # embeddings are normalised in `get_prompt_embeddings`,
+ # like Gemma 3. For older versions, we need to manually normalise.
+ embed_scale = hf_model.config.hidden_size**0.5
+ normalizer = torch.tensor(embed_scale, dtype=embed.dtype)
+ embed *= normalizer
+
+ # MiniCPM models apply scale_emb to embeddings internally.
+ # vLLM expects pre-scaled embeddings when using inputs_embeds.
+ if model in EMBED_SCALING_MODELS:
+ config = hf_model.model.config
+ embed = embed * config.scale_emb
+
+ prompt_embeds.append(embed.squeeze(0))
+
+ with vllm_runner(
+ model,
+ tokenizer_name=model_info.tokenizer or model,
+ tokenizer_mode=model_info.tokenizer_mode,
+ revision=model_info.revision,
+ trust_remote_code=model_info.trust_remote_code,
+ # Remove the effects of batch variance on ROCm since batch invariance
+ # is not yet supported.
+ # See: https://github.com/vllm-project/vllm/issues/27433
+ max_num_seqs=1 if current_platform.is_rocm() else 2,
+ enable_prompt_embeds=use_prompt_embeds,
+ compilation_config={"cudagraph_capture_sizes": [1, 2]},
+ ) as vllm_model:
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
+ example_prompts, max_tokens, num_logprobs
+ )
+ if prompt_embeds is not None:
+ vllm_outputs_from_embeds = vllm_model.generate_greedy_logprobs(
+ prompt_embeds, max_tokens, num_logprobs
+ )
+
+ check_logprobs_close(
+ outputs_0_lst=hf_outputs,
+ outputs_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ )
+ if prompt_embeds is not None:
+ check_logprobs_close(
+ outputs_0_lst=vllm_outputs,
+ outputs_1_lst=vllm_outputs_from_embeds,
+ name_0="vllm",
+ name_1="vllm_from_embeds",
+ )
+
+ if use_rocm_aiter:
+ # this is to ensure that vllm engine
+ # has deallocated the memory before running the next
+ # unit tests. On ROCm, when using AITER
+ # the memory might not be deallocated completely
+ # before running the next test case
+ torch.accelerator.synchronize()
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation/test_gemma.py b/asian_ai/performance/vLLM/tests/models/language/generation/test_gemma.py
new file mode 100644
index 0000000000000000000000000000000000000000..cf2ebd9a613a3d71ffb0e23cebe0f3ec1f50589a
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/generation/test_gemma.py
@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import numpy as np
+import pytest
+
+MODELS = ["google/gemma-2b", "google/gemma-2-2b", "google/gemma-3-4b-it"]
+
+
+@pytest.mark.parametrize("model", MODELS)
+def test_dummy_loader(vllm_runner, monkeypatch, model: str) -> None:
+ with monkeypatch.context() as m:
+ m.setenv("VLLM_ALLOW_INSECURE_SERIALIZATION", "1")
+ with vllm_runner(
+ model,
+ load_format="dummy",
+ ) as llm:
+ if model == "google/gemma-3-4b-it":
+ normalizers = llm.llm.collective_rpc(
+ lambda self: self.model_runner.model.language_model.model.normalizer.cpu().item() # noqa: E501
+ )
+ config = llm.llm.llm_engine.model_config.hf_config.text_config
+ else:
+ normalizers = llm.llm.collective_rpc(
+ lambda self: self.model_runner.model.model.normalizer.cpu().item()
+ )
+ config = llm.llm.llm_engine.model_config.hf_config
+ assert np.allclose(normalizers, config.hidden_size**0.5, rtol=2e-3)
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation/test_granite.py b/asian_ai/performance/vLLM/tests/models/language/generation/test_granite.py
new file mode 100644
index 0000000000000000000000000000000000000000..10e344b0d7b34874bfe240264e9b263b8a9df5cf
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/generation/test_granite.py
@@ -0,0 +1,42 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+
+from ...utils import check_logprobs_close
+
+MODELS = [
+ # TODO(sang): Sliding window should be tested separately.
+ "ibm/PowerLM-3b",
+ "ibm/PowerMoE-3b",
+]
+
+
+@pytest.mark.parametrize("model", MODELS)
+@pytest.mark.parametrize("dtype", ["bfloat16"])
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("num_logprobs", [5])
+@pytest.mark.cpu_model
+def test_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+ max_tokens: int,
+ num_logprobs: int,
+) -> None:
+ with hf_runner(model, dtype=dtype) as hf_model:
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ with vllm_runner(model, dtype=dtype) as vllm_model:
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
+ example_prompts, max_tokens, num_logprobs
+ )
+ check_logprobs_close(
+ outputs_0_lst=hf_outputs,
+ outputs_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation/test_hybrid.py b/asian_ai/performance/vLLM/tests/models/language/generation/test_hybrid.py
new file mode 100644
index 0000000000000000000000000000000000000000..10d2601a1a6b830ca0ab0f64e65c30db83d52ec7
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/generation/test_hybrid.py
@@ -0,0 +1,903 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from collections.abc import Callable
+from contextlib import contextmanager, nullcontext
+
+import pytest
+
+from tests.models.registry import HF_EXAMPLE_MODELS
+from tests.utils import multi_gpu_test
+from vllm import LLM
+from vllm.engine.arg_utils import EngineArgs
+from vllm.platforms import current_platform
+from vllm.sampling_params import SamplingParams
+from vllm.v1.cudagraph_dispatcher import CudagraphDispatcher
+
+from ...utils import check_logprobs_close, check_outputs_equal
+
+# Mark all tests as hybrid
+pytestmark = pytest.mark.hybrid_model
+
+# NOTE: The first model in each list is taken as the primary model,
+# meaning that it will be used in all tests in this file
+# The rest of the models will only be tested by test_models
+
+APC_MULTIPLY_BY = 300
+
+SSM_MODELS = [
+ "state-spaces/mamba-130m-hf",
+ "tiiuae/falcon-mamba-tiny-dev",
+ # mamba2-codestral in transformers is broken pending:
+ # https://github.com/huggingface/transformers/pull/40861
+ # "yujiepan/mamba2-codestral-v0.1-tiny-random",
+]
+
+HYBRID_MODELS = [
+ "ai21labs/Jamba-tiny-dev",
+ "pfnet/plamo-2-1b",
+ "Zyphra/Zamba2-1.2B-instruct",
+ "ibm-granite/granite-4.0-tiny-preview",
+ "tiiuae/Falcon-H1-0.5B-Base",
+ "LiquidAI/LFM2-1.2B",
+ "tiny-random/qwen3-next-moe",
+]
+
+FULL_CUDA_GRAPH_MODELS = [
+ "ai21labs/Jamba-tiny-dev",
+ "pfnet/plamo-2-1b",
+ "Zyphra/Zamba2-1.2B-instruct",
+]
+
+FP32_STATE_MODELS = [
+ "state-spaces/mamba-130m-hf",
+ "Zyphra/Zamba2-1.2B-instruct",
+]
+
+# Avoid OOM
+MAX_NUM_SEQS = 4
+
+ATTN_BACKEND = "TRITON_ATTN" if current_platform.is_rocm() else "auto"
+
+
+def _set_conv_state_layout(monkeypatch, layout: str) -> None:
+ """Set conv state layout env var and clear cache to pick up new value."""
+ from vllm.model_executor.layers.mamba import mamba_utils
+
+ monkeypatch.setenv("VLLM_SSM_CONV_STATE_LAYOUT", layout)
+ mamba_utils.get_conv_state_layout.cache_clear()
+
+
+@pytest.mark.parametrize("model", SSM_MODELS + HYBRID_MODELS)
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("num_logprobs", [5])
+def test_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ monkeypatch,
+ model: str,
+ max_tokens: int,
+ num_logprobs: int,
+) -> None:
+ try:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+ except ValueError:
+ pass
+
+ with hf_runner(model) as hf_model:
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ with vllm_runner(
+ model, max_num_seqs=MAX_NUM_SEQS, attention_backend=ATTN_BACKEND
+ ) as vllm_model:
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ check_logprobs_close(
+ outputs_0_lst=hf_outputs,
+ outputs_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ )
+
+
+@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("num_logprobs", [5])
+@pytest.mark.parametrize("conv_state_layout", ["SD", "DS"])
+def test_batching(
+ vllm_runner,
+ example_prompts,
+ monkeypatch,
+ model: str,
+ max_tokens: int,
+ num_logprobs: int,
+ conv_state_layout: str,
+) -> None:
+ try:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+ except ValueError:
+ pass
+
+ _set_conv_state_layout(monkeypatch, conv_state_layout)
+
+ for_loop_outputs = []
+ with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:
+ for prompt in example_prompts:
+ (single_output,) = vllm_model.generate_greedy_logprobs(
+ [prompt], max_tokens, num_logprobs
+ )
+ for_loop_outputs.append(single_output)
+
+ batched_outputs = vllm_model.generate_greedy_logprobs(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ check_logprobs_close(
+ outputs_0_lst=for_loop_outputs,
+ outputs_1_lst=batched_outputs,
+ name_0="for_loop_vllm",
+ name_1="batched_vllm",
+ )
+
+
+@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
+@pytest.mark.parametrize("max_tokens", [10])
+@pytest.mark.parametrize("conv_state_layout", ["SD", "DS"])
+def test_chunked_prefill_with_parallel_sampling(
+ vllm_runner,
+ example_prompts,
+ monkeypatch,
+ model: str,
+ max_tokens: int,
+ conv_state_layout: str,
+) -> None:
+ """
+ Tests chunked prefill in conjunction with n > 1.
+
+ In this case, prefill is populated with decoding tokens and
+ we test that it doesn't fail.
+
+ This test might fail if cache is not allocated correctly for n > 1
+ decoding steps inside a chunked prefill forward pass
+ (where we have both prefill and decode together)
+ """
+ _set_conv_state_layout(monkeypatch, conv_state_layout)
+
+ sampling_params = SamplingParams(n=3, temperature=1, seed=0, max_tokens=max_tokens)
+ with vllm_runner(
+ model,
+ enable_chunked_prefill=True,
+ # forces prefill chunks with decoding
+ max_num_batched_tokens=MAX_NUM_SEQS * 3,
+ max_num_seqs=MAX_NUM_SEQS,
+ attention_backend=ATTN_BACKEND,
+ ) as vllm_model:
+ vllm_model.generate(example_prompts, sampling_params)
+
+
+@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
+@pytest.mark.parametrize("max_tokens", [20])
+@pytest.mark.parametrize("conv_state_layout", ["SD", "DS"])
+def test_mamba_cache_cg_padding(
+ vllm_runner,
+ example_prompts,
+ monkeypatch,
+ model: str,
+ max_tokens: int,
+ conv_state_layout: str,
+) -> None:
+ """
+ This test is for verifying that mamba cache is padded to CG captured
+ batch size. If it's not, a torch RuntimeError will be raised because
+ tensor dimensions aren't compatible.
+ """
+ _set_conv_state_layout(monkeypatch, conv_state_layout)
+
+ vllm_config = EngineArgs(model=model, trust_remote_code=True).create_engine_config()
+ cudagraph_dispatcher = CudagraphDispatcher(vllm_config)
+ cudagraph_dispatcher.initialize_cudagraph_keys(
+ vllm_config.compilation_config.cudagraph_mode
+ )
+ while (
+ len(example_prompts)
+ == cudagraph_dispatcher.dispatch(len(example_prompts))[1].num_tokens
+ ):
+ example_prompts.append(example_prompts[0])
+
+ try:
+ with vllm_runner(model) as vllm_model:
+ vllm_model.generate_greedy(example_prompts, max_tokens)
+ except RuntimeError:
+ pytest.fail(
+ "Couldn't run batch size which is not equal to a Cuda Graph "
+ "captured batch size. "
+ "Could be related to mamba cache not padded correctly"
+ )
+
+
+@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
+def test_fail_upon_inc_requests_and_finished_requests_lt_available_blocks(
+ vllm_runner,
+ example_prompts,
+ model: str,
+) -> None:
+ """
+ This test is for verifying that the hybrid inner state management doesn't
+ collapse in case where the number of incoming requests and
+ finished_requests_ids is larger than the maximum mamba block capacity.
+
+ This could generally happen due to the fact that hybrid does support
+ statelessness mechanism where it can clean up new incoming requests in
+ a single step.
+ """
+ try:
+ with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:
+ vllm_model.generate_greedy([example_prompts[0]] * 100, 10)
+ except ValueError:
+ pytest.fail(
+ "Hybrid inner state wasn't cleaned up properly between"
+ "steps finished requests registered unnecessarily "
+ )
+
+
+@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
+def test_state_cleanup(
+ vllm_runner,
+ example_prompts,
+ model: str,
+) -> None:
+ """
+ This test is for verifying that the Hybrid state is cleaned up between
+ steps.
+
+ If it's not cleaned, an error would be expected.
+ """
+ try:
+ with vllm_runner(model, max_num_seqs=MAX_NUM_SEQS) as vllm_model:
+ for _ in range(10):
+ vllm_model.generate_greedy([example_prompts[0]] * 100, 1)
+ except ValueError:
+ pytest.fail(
+ "Hybrid inner state wasn't cleaned up between states, "
+ "could be related to finished_requests_ids"
+ )
+
+
+@multi_gpu_test(num_gpus=2)
+@pytest.mark.parametrize("model", [SSM_MODELS[0], HYBRID_MODELS[0]])
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("num_logprobs", [5])
+def test_distributed_correctness(
+ vllm_runner,
+ example_prompts,
+ model: str,
+ max_tokens: int,
+ num_logprobs: int,
+) -> None:
+ with vllm_runner(
+ model, tensor_parallel_size=1, max_num_seqs=MAX_NUM_SEQS
+ ) as vllm_model:
+ vllm_outputs_tp_1 = vllm_model.generate_greedy_logprobs(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ with vllm_runner(
+ model, tensor_parallel_size=2, max_num_seqs=MAX_NUM_SEQS
+ ) as vllm_model:
+ vllm_outputs_tp_2 = vllm_model.generate_greedy_logprobs(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ check_logprobs_close(
+ outputs_0_lst=vllm_outputs_tp_1,
+ outputs_1_lst=vllm_outputs_tp_2,
+ name_0="vllm_tp_1",
+ name_1="vllm_tp_2",
+ )
+
+
+@pytest.mark.parametrize("model", FULL_CUDA_GRAPH_MODELS)
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("num_logprobs", [5])
+def test_full_cuda_graph(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ monkeypatch,
+ model: str,
+ max_tokens: int,
+ num_logprobs: int,
+) -> None:
+ try:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+ except ValueError:
+ pass
+
+ with hf_runner(model) as hf_model:
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ with vllm_runner(
+ model, max_num_seqs=MAX_NUM_SEQS, attention_backend=ATTN_BACKEND
+ ) as vllm_model:
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ check_logprobs_close(
+ outputs_0_lst=hf_outputs,
+ outputs_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ )
+
+
+@pytest.mark.parametrize("model", FP32_STATE_MODELS)
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("num_logprobs", [5])
+@pytest.mark.parametrize(
+ "cache_dtype_param", ["mamba_ssm_cache_dtype", "mamba_cache_dtype"]
+)
+def test_fp32_cache_state(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ monkeypatch,
+ model: str,
+ max_tokens: int,
+ num_logprobs: int,
+ cache_dtype_param: str,
+) -> None:
+ try:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+ except ValueError:
+ pass
+
+ with hf_runner(model) as hf_model:
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ with vllm_runner(
+ model, max_num_seqs=MAX_NUM_SEQS, **{cache_dtype_param: "float32"}
+ ) as vllm_model:
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ check_logprobs_close(
+ outputs_0_lst=hf_outputs,
+ outputs_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ )
+
+
+# Helper functions for the APC tests
+def _get_vllm_runner_params(
+ model: str,
+ max_model_len: int,
+ tensor_parallel_size: int = 1,
+):
+ return {
+ "model_name": model,
+ "enable_chunked_prefill": True,
+ "enable_prefix_caching": False,
+ "max_model_len": max_model_len,
+ "tensor_parallel_size": tensor_parallel_size,
+ "gpu_memory_utilization": 0.4,
+ "attention_backend": ATTN_BACKEND,
+ }
+
+
+@contextmanager
+def _owned_vllm_runner(vllm_runner, kwargs):
+ with vllm_runner(**kwargs) as runner:
+ yield runner
+
+
+def _get_vLLM_output(
+ vllm_runner,
+ kwargs,
+ prompts,
+ max_tokens,
+ num_logprobs,
+ num_repetitions=1,
+ vllm_model=None,
+):
+ runner_context = (
+ _owned_vllm_runner(vllm_runner, kwargs)
+ if vllm_model is None
+ else nullcontext(vllm_model)
+ )
+ with runner_context as runner:
+ outs = []
+ for _ in range(num_repetitions):
+ if num_logprobs < 0:
+ vllm_output = runner.generate_greedy(prompts, max_tokens)
+ else:
+ vllm_output = runner.generate_greedy_logprobs(
+ prompts, max_tokens, num_logprobs
+ )
+ outs.append(vllm_output)
+
+ return outs, vllm_model
+
+
+@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("n_repetitions", [2])
+# If num_logprobs is set to -1, then the stringent version
+# of the test is executed using `check_outputs_equal`
+# instead of `check_logprobs_close`
+@pytest.mark.parametrize("num_logprobs", [5])
+@pytest.mark.parametrize("tensor_parallel_size", [1])
+def test_apc_single_prompt(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ monkeypatch,
+ model: str,
+ max_tokens: int,
+ n_repetitions: int,
+ num_logprobs: int,
+ tensor_parallel_size: int,
+) -> None:
+ try:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+ except ValueError:
+ pass
+
+ compare_operator: Callable = (
+ check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
+ )
+
+ # Sample prompts.
+ generated_prompts = [APC_MULTIPLY_BY * example_prompts[0]]
+
+ max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
+ vllm_runner_kwargs = _get_vllm_runner_params(
+ model, max_model_len, tensor_parallel_size=tensor_parallel_size
+ )
+ vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
+ vllm_outputs_no_cache, _ = _get_vLLM_output(
+ vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
+ )
+
+ vllm_runner_kwargs["enable_prefix_caching"] = True
+ vllm_outputs_cache_rep, _ = _get_vLLM_output(
+ vllm_runner,
+ vllm_runner_kwargs,
+ generated_prompts,
+ max_tokens,
+ num_logprobs,
+ n_repetitions,
+ )
+
+ for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
+ # In the first repetition, the caches are filled
+ # In the second repetition, these caches are reused
+
+ compare_operator(
+ outputs_0_lst=vllm_outputs_no_cache[0],
+ outputs_1_lst=vllm_outputs_cache_itn,
+ name_0="vllm_no_cache",
+ name_1=f"vllm_cache_it_{r_idx + 1}",
+ )
+
+
+@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("n_repetitions", [2])
+# If num_logprobs is set to -1, then the stringent version
+# of the test is executed using `check_outputs_equal`
+# instead of `check_logprobs_close`
+@pytest.mark.parametrize("num_logprobs", [5])
+@pytest.mark.parametrize("tensor_parallel_size", [1])
+def test_apc_single_prompt_block_align_alignment(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ monkeypatch,
+ model: str,
+ max_tokens: int,
+ n_repetitions: int,
+ num_logprobs: int,
+ tensor_parallel_size: int,
+) -> None:
+ try:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+ except ValueError:
+ pass
+
+ compare_operator: Callable = (
+ check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
+ )
+
+ # Sample prompts. This custom prompt is used, as it causes the most issues
+ generated_prompts = ["The president of the United States is " * APC_MULTIPLY_BY]
+
+ max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
+ vllm_runner_kwargs = _get_vllm_runner_params(
+ model, max_model_len, tensor_parallel_size=tensor_parallel_size
+ )
+ vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
+
+ vllm_outputs_no_cache, _ = _get_vLLM_output(
+ vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
+ )
+
+ vllm_runner_kwargs["enable_prefix_caching"] = True
+ with vllm_runner(**vllm_runner_kwargs) as vllm_model:
+ # Retrieve the default mamba state block size
+ vllm_config = vllm_model.llm.llm_engine.vllm_config
+ mamba_block_size = vllm_config.cache_config.mamba_block_size
+
+ # In case the hybrid model does not have the
+ # "mamba_block_size" assume a fixed constant
+ if mamba_block_size is None:
+ mamba_block_size = 512
+
+ mamba_block_size_multiplier = 10
+ for offsets in [-3, 3, mamba_block_size // 4 + 3, mamba_block_size // 2 - 3]:
+ vllm_runner_kwargs["max_num_batched_tokens"] = (
+ mamba_block_size_multiplier * mamba_block_size - offsets
+ )
+ vllm_outputs_cache_rep, _ = _get_vLLM_output(
+ vllm_runner,
+ vllm_runner_kwargs,
+ generated_prompts,
+ max_tokens,
+ num_logprobs,
+ n_repetitions,
+ )
+
+ # Check alignment of the output logits when using APC
+ for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
+ # In the first repetition, the caches are filled
+ # In the second repetition, these caches are reused
+
+ compare_operator(
+ outputs_0_lst=vllm_outputs_no_cache[0],
+ outputs_1_lst=vllm_outputs_cache_itn,
+ name_0="vllm_no_cache",
+ name_1=f"vllm_cache_it_{r_idx + 1}",
+ )
+
+
+@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("n_repetitions", [2])
+# If num_logprobs is set to -1, then the stringent version
+# of the test is executed using `check_outputs_equal`
+# instead of `check_logprobs_close`
+@pytest.mark.parametrize("num_logprobs", [5])
+@pytest.mark.parametrize("tensor_parallel_size", [1])
+def test_apc_multiple_prompts_all_cached_outputs(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ monkeypatch,
+ model: str,
+ max_tokens: int,
+ n_repetitions: int,
+ num_logprobs: int,
+ tensor_parallel_size: int,
+) -> None:
+ try:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+ except ValueError:
+ pass
+
+ compare_operator: Callable = (
+ check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
+ )
+
+ # Sample prompts.
+ generated_prompts = [APC_MULTIPLY_BY * prompt for prompt in example_prompts]
+
+ max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
+ vllm_runner_kwargs = _get_vllm_runner_params(
+ model, max_model_len, tensor_parallel_size=tensor_parallel_size
+ )
+ vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
+ # Reduce the effects of batch variance on ROCm since batch invariance is not
+ # yet supported. See: https://github.com/vllm-project/vllm/issues/27433
+ if current_platform.is_rocm():
+ vllm_runner_kwargs["max_num_seqs"] = 4
+
+ vllm_outputs_no_cache, _ = _get_vLLM_output(
+ vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
+ )
+
+ vllm_runner_kwargs["enable_prefix_caching"] = True
+ vllm_outputs_cache_rep, _ = _get_vLLM_output(
+ vllm_runner,
+ vllm_runner_kwargs,
+ generated_prompts,
+ max_tokens,
+ num_logprobs,
+ n_repetitions,
+ )
+
+ for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
+ # In the first repetition, the caches are filled
+ # In the second repetition, these caches are reused
+
+ compare_operator(
+ outputs_0_lst=vllm_outputs_no_cache[0],
+ outputs_1_lst=vllm_outputs_cache_itn,
+ name_0="vllm_no_cache",
+ name_1=f"vllm_cache_it_{r_idx + 1}",
+ )
+
+
+@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("n_repetitions", [2])
+# If num_logprobs is set to -1, then the stringent version
+# of the test is executed using `check_outputs_equal`
+# instead of `check_logprobs_close`
+@pytest.mark.parametrize("num_logprobs", [5])
+@pytest.mark.parametrize("tensor_parallel_size", [1])
+def test_apc_multiple_prompts_block_align_alignment(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ monkeypatch,
+ model: str,
+ max_tokens: int,
+ n_repetitions: int,
+ num_logprobs: int,
+ tensor_parallel_size: int,
+) -> None:
+ try:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+ except ValueError:
+ pass
+
+ compare_operator: Callable = (
+ check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
+ )
+
+ # Sample prompts. This custom prompt is used, as it causes the most issues
+ prompt_text = "The president of the United States is "
+ prompt_offsets = [0, 3, 7, 13, 17, 22, 25, 31]
+ generated_prompts = [
+ prompt_text[offset:] * APC_MULTIPLY_BY for offset in prompt_offsets
+ ]
+
+ max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
+ vllm_runner_kwargs = _get_vllm_runner_params(
+ model, max_model_len, tensor_parallel_size
+ )
+ vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
+
+ vllm_outputs_no_cache, _ = _get_vLLM_output(
+ vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
+ )
+
+ vllm_runner_kwargs["enable_prefix_caching"] = True
+ with vllm_runner(**vllm_runner_kwargs) as vllm_model:
+ # Retrieve the default mamba state block size
+ vllm_config = vllm_model.llm.llm_engine.vllm_config
+ mamba_block_size = vllm_config.cache_config.mamba_block_size
+
+ # In case the hybrid model does not have the
+ # "mamba_block_size" assume a fixed constant
+ if mamba_block_size is None:
+ mamba_block_size = 512
+
+ mamba_block_size_multiplier = 10
+ for offsets in [-3, 3, mamba_block_size // 4 + 3, mamba_block_size // 2 - 3]:
+ vllm_runner_kwargs["max_num_batched_tokens"] = (
+ mamba_block_size_multiplier * mamba_block_size - offsets
+ )
+ vllm_outputs_cache_rep, _ = _get_vLLM_output(
+ vllm_runner,
+ vllm_runner_kwargs,
+ generated_prompts,
+ max_tokens,
+ num_logprobs,
+ n_repetitions,
+ )
+
+ # Check alignment of the output logits when using APC
+ for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
+ # In the first repetition, the caches are filled
+ # In the second repetition, these caches are reused
+
+ compare_operator(
+ outputs_0_lst=vllm_outputs_no_cache[0],
+ outputs_1_lst=vllm_outputs_cache_itn,
+ name_0="vllm_no_cache",
+ name_1=f"vllm_cache_it_{r_idx + 1}",
+ )
+
+
+@pytest.mark.parametrize("model", [HYBRID_MODELS[0]])
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("n_repetitions", [2])
+# If num_logprobs is set to -1, then the stringent version
+# of the test is executed using `check_outputs_equal`
+# instead of `check_logprobs_close`
+@pytest.mark.parametrize("num_logprobs", [5])
+@pytest.mark.parametrize("tensor_parallel_size", [1])
+def test_apc_multiple_prompts_partial_cached_outputs(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ monkeypatch,
+ model: str,
+ max_tokens: int,
+ n_repetitions: int,
+ num_logprobs: int,
+ tensor_parallel_size: int,
+) -> None:
+ try:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_available_online(on_fail="skip")
+ model_info.check_transformers_version(on_fail="skip")
+ except ValueError:
+ pass
+
+ compare_operator: Callable = (
+ check_logprobs_close if num_logprobs > 0 else check_outputs_equal # type: ignore
+ )
+
+ # Sample prompts.
+ generated_prompts = [APC_MULTIPLY_BY * prompt for prompt in example_prompts]
+
+ max_model_len = max(len(prompt) + max_tokens for prompt in generated_prompts)
+ vllm_runner_kwargs = _get_vllm_runner_params(
+ model, max_model_len, tensor_parallel_size=tensor_parallel_size
+ )
+ vllm_runner_kwargs["mamba_ssm_cache_dtype"] = "float32"
+
+ vllm_outputs_no_cache, _ = _get_vLLM_output(
+ vllm_runner, vllm_runner_kwargs, generated_prompts, max_tokens, num_logprobs
+ )
+
+ # Cache only part of all the prompts
+ vllm_runner_kwargs["enable_prefix_caching"] = True
+ with _owned_vllm_runner(vllm_runner, vllm_runner_kwargs) as vllm_model:
+ vllm_outputs_partial_cache, _ = _get_vLLM_output(
+ vllm_runner,
+ vllm_runner_kwargs,
+ generated_prompts[:3],
+ max_tokens,
+ num_logprobs,
+ vllm_model=vllm_model,
+ )
+
+ compare_operator(
+ outputs_0_lst=vllm_outputs_no_cache[0][:3],
+ outputs_1_lst=vllm_outputs_partial_cache[0],
+ name_0="vllm_no_cache",
+ name_1="vllm_partial_cache",
+ )
+
+ vllm_outputs_cache_rep, _ = _get_vLLM_output(
+ vllm_runner,
+ vllm_runner_kwargs,
+ generated_prompts,
+ max_tokens,
+ num_logprobs,
+ n_repetitions,
+ vllm_model=vllm_model,
+ )
+
+ for r_idx, vllm_outputs_cache_itn in enumerate(vllm_outputs_cache_rep):
+ # In the first repetition, the caches are filled
+ # In the second repetition, these caches are reused
+
+ compare_operator(
+ outputs_0_lst=vllm_outputs_no_cache[0],
+ outputs_1_lst=vllm_outputs_cache_itn,
+ name_0="vllm_no_cache",
+ name_1=f"vllm_cache_it_{r_idx + 1}",
+ )
+
+
+# Test that outputs match whether prefix caching is enabled or not for mamba.
+@pytest.mark.parametrize("model", ["tiiuae/falcon-mamba-7b"])
+def test_same_mamba_output_apc_on_vs_off(
+ vllm_runner,
+ model: str,
+) -> None:
+ num_logprobs = 5
+ prompts = [
+ "hello what is one plus one what is one plus one what is one plus one the answer is", # noqa: E501
+ "hello what is one plus one what is one plus one what is one plus one the answer is", # noqa: E501
+ ]
+ max_tokens = 20
+ max_model_len = max(len(p) for p in prompts) + max_tokens + 64
+
+ base_kwargs = _get_vllm_runner_params(model, max_model_len)
+ base_kwargs.update(
+ enforce_eager=True, block_size=16, seed=42, gpu_memory_utilization=0.8
+ )
+
+ # No prefix caching
+ kwargs_no_apc = {**base_kwargs, "enable_prefix_caching": False}
+ with vllm_runner(**kwargs_no_apc) as vllm_model:
+ outputs_no_apc, _ = _get_vLLM_output(
+ vllm_runner,
+ kwargs_no_apc,
+ prompts,
+ max_tokens,
+ num_logprobs=num_logprobs,
+ vllm_model=vllm_model,
+ )
+ # With prefix caching
+ kwargs_with_apc = {
+ **base_kwargs,
+ "enable_prefix_caching": True,
+ "mamba_block_size": 16,
+ }
+ with vllm_runner(**kwargs_with_apc) as vllm_model:
+ outputs_with_apc, _ = _get_vLLM_output(
+ vllm_runner,
+ kwargs_with_apc,
+ prompts,
+ max_tokens,
+ num_logprobs=num_logprobs,
+ vllm_model=vllm_model,
+ )
+
+ check_logprobs_close(
+ outputs_0_lst=outputs_no_apc[0],
+ outputs_1_lst=outputs_with_apc[0],
+ name_0="vllm_no_apc",
+ name_1="vllm_with_apc",
+ )
+
+
+# we have to use a real large model to get reasonable results
+# the model can't be a hybrid model as we need block_size 16
+@pytest.mark.parametrize("model", ["tiiuae/falcon-mamba-7b"])
+def test_apc_common_prefix_same_batch(
+ model: str,
+ monkeypatch,
+) -> None:
+ # Required to put the two requests in the same batch
+ monkeypatch.setenv("VLLM_ENABLE_V1_MULTIPROCESSING", "0")
+ llm = LLM(
+ model=model,
+ enforce_eager=True,
+ block_size=16,
+ mamba_block_size=16,
+ enable_prefix_caching=True,
+ seed=42,
+ attention_backend=ATTN_BACKEND,
+ )
+ prompts = [
+ "hello what is one plus one what is one plus one what is one plus one the answer is", # noqa: E501
+ "hello what is one plus one what is one plus one what is one plus one the answer is", # noqa: E501
+ ]
+ sampling_params = SamplingParams(temperature=0.0, max_tokens=20)
+ outputs = llm.generate(prompts, sampling_params)
+ for output in outputs:
+ assert "two" in output.outputs[0].text
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation/test_mistral.py b/asian_ai/performance/vLLM/tests/models/language/generation/test_mistral.py
new file mode 100644
index 0000000000000000000000000000000000000000..2dcda1bd6088c54323b7c82b484e4868474ad5e2
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/generation/test_mistral.py
@@ -0,0 +1,352 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import copy
+import json
+
+import pytest
+
+from vllm.sampling_params import SamplingParams
+from vllm.tokenizers.mistral import MistralTokenizer
+from vllm.tool_parsers.mistral_tool_parser import (
+ MistralToolCall,
+ MistralToolParser,
+)
+
+from ...utils import check_logprobs_close
+
+MODELS = [
+ "mistralai/Mistral-7B-Instruct-v0.3",
+]
+
+MISTRAL_FORMAT_MODELS = [
+ "mistralai/Mistral-7B-Instruct-v0.3",
+ # uses the v3-Tekken tokenizer
+ "mistralai/Ministral-8B-Instruct-2410",
+ # Mistral-Nemo is too big for CI, but passes locally
+ # "mistralai/Mistral-Nemo-Instruct-2407"
+]
+
+SAMPLING_PARAMS = SamplingParams(max_tokens=512, temperature=0.0, logprobs=5)
+SYMBOLIC_LANG_PROMPTS = [
+ "勇敢な船乗りについての詩を書く", # japanese
+ "寫一首關於勇敢的水手的詩", # chinese
+ "ပုံပြင်လေးပြောပြပါ်:\n", # burmese
+ "Repeat the phrase 'URGENCY🌶️':\nURGENCY🌶️\nURGENCY🌶️\n", # see https://github.com/vllm-project/vllm/pull/9625
+]
+
+# for function calling
+TOOLS = [
+ {
+ "type": "function",
+ "function": {
+ "name": "get_current_weather",
+ "description": "Get the current weather in a given location",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "city": {
+ "type": "string",
+ "description": "The city to find the weather for, e.g. "
+ "'San Francisco'",
+ },
+ "state": {
+ "type": "string",
+ "description": "the two-letter abbreviation for the state that "
+ "the city is in, e.g. 'CA' which would mean 'California'",
+ },
+ "unit": {
+ "type": "string",
+ "description": "The unit to fetch the temperature in",
+ "enum": ["celsius", "fahrenheit"],
+ },
+ },
+ "required": ["city", "state", "unit"],
+ },
+ },
+ },
+ {
+ "type": "function",
+ "function": {
+ "name": "rewrite",
+ "description": "Rewrites text",
+ "parameters": {
+ "type": "object",
+ "required": [],
+ "properties": {
+ "text": {
+ "type": "string",
+ "description": "The input text to rewrite.",
+ }
+ },
+ },
+ },
+ },
+]
+MSGS = [
+ {"role": "system", "content": "You are an assistant."},
+ {
+ "role": "user",
+ "content": "Could you please rewrite the below article? \n\n My English needs "
+ "improving, maybe I make errors.",
+ },
+ {
+ "role": "assistant",
+ "content": "",
+ "tool_calls": [
+ {
+ "id": "bbc5b7ede",
+ "type": "function",
+ "function": {
+ "name": "rewrite",
+ "arguments": '{"text":"My English needs improving, maybe '
+ 'I make errors."}',
+ },
+ }
+ ],
+ },
+ {
+ "role": "tool",
+ "content": '{"action":"rewrite","outcome":"My English needs improving, maybe '
+ 'I make errors."}',
+ "tool_call_id": "bbc5b7ede",
+ "name": "rewrite",
+ },
+ {
+ "role": "assistant",
+ "content": "---\n\nMy English needs improving, maybe I make errors",
+ },
+ {
+ "role": "user",
+ "content": (
+ "Can you tell me what the temperate will be in Dallas, in fahrenheit?"
+ ),
+ },
+]
+
+SAMPLE_JSON_SCHEMA = {
+ "type": "object",
+ "properties": {
+ "name": {"type": "string"},
+ "age": {"type": "integer"},
+ "skills": {
+ "type": "array",
+ "items": {"type": "string", "maxLength": 10},
+ "minItems": 3,
+ },
+ "work_history": {
+ "type": "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "company": {"type": "string"},
+ "duration": {"type": "number"},
+ "position": {"type": "string"},
+ },
+ "required": ["company", "position"],
+ },
+ },
+ },
+ "required": ["name", "age", "skills", "work_history"],
+}
+
+
+@pytest.mark.parametrize("model", MODELS)
+@pytest.mark.parametrize("dtype", ["bfloat16"])
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("num_logprobs", [5])
+def test_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+ max_tokens: int,
+ num_logprobs: int,
+) -> None:
+ # TODO(sang): Sliding window should be tested separately.
+ with hf_runner(model, dtype=dtype) as hf_model:
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ with vllm_runner(model, dtype=dtype, tokenizer_mode="mistral") as vllm_model:
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ check_logprobs_close(
+ outputs_0_lst=hf_outputs,
+ outputs_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ )
+
+
+@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
+@pytest.mark.parametrize("dtype", ["bfloat16"])
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("num_logprobs", [5])
+def test_mistral_format(
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+ max_tokens: int,
+ num_logprobs: int,
+) -> None:
+ with vllm_runner(
+ model,
+ dtype=dtype,
+ tokenizer_mode="mistral",
+ load_format="mistral",
+ config_format="mistral",
+ ) as mistral_format_model:
+ mistral_format_outputs = mistral_format_model.generate_greedy_logprobs(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ with vllm_runner(
+ model,
+ dtype=dtype,
+ tokenizer_mode="hf",
+ load_format="safetensors",
+ config_format="hf",
+ ) as hf_format_model:
+ hf_format_outputs = hf_format_model.generate_greedy_logprobs(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ check_logprobs_close(
+ outputs_0_lst=hf_format_outputs,
+ outputs_1_lst=mistral_format_outputs,
+ name_0="hf",
+ name_1="mistral",
+ )
+
+
+@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
+@pytest.mark.parametrize("dtype", ["bfloat16"])
+def test_mistral_symbolic_languages(vllm_runner, model: str, dtype: str) -> None:
+ with vllm_runner(
+ model,
+ dtype=dtype,
+ max_model_len=8192,
+ tokenizer_mode="mistral",
+ config_format="mistral",
+ load_format="mistral",
+ ) as vllm_model:
+ for prompt in SYMBOLIC_LANG_PROMPTS:
+ msg = {"role": "user", "content": prompt}
+ outputs = vllm_model.llm.chat([msg], sampling_params=SAMPLING_PARAMS)
+ assert "�" not in outputs[0].outputs[0].text.strip()
+
+
+@pytest.mark.parametrize("model", MISTRAL_FORMAT_MODELS)
+@pytest.mark.parametrize("dtype", ["bfloat16"])
+def test_mistral_function_calling(vllm_runner, model: str, dtype: str) -> None:
+ with vllm_runner(
+ model,
+ dtype=dtype,
+ tokenizer_mode="mistral",
+ config_format="mistral",
+ load_format="mistral",
+ ) as vllm_model:
+ msgs = copy.deepcopy(MSGS)
+ outputs = vllm_model.llm.chat(
+ msgs, tools=TOOLS, sampling_params=SAMPLING_PARAMS
+ )
+
+ tokenizer = vllm_model.llm.get_tokenizer()
+ tool_parser = MistralToolParser(tokenizer)
+
+ model_output = outputs[0].outputs[0].text.strip()
+ assert model_output.startswith(tool_parser.bot_token), model_output
+ parsed_message = tool_parser.extract_tool_calls(model_output, None)
+
+ assert parsed_message.tools_called
+
+ assert MistralToolCall.is_valid_id(parsed_message.tool_calls[0].id)
+ assert parsed_message.tool_calls[0].function.name == "get_current_weather"
+ assert (
+ parsed_message.tool_calls[0].function.arguments
+ == '{"city": "Dallas", "state": "TX", "unit": "fahrenheit"}'
+ ) # noqa
+ assert parsed_message.content is None
+
+
+def test_mistral_function_call_nested_json():
+ """Ensure that the function-name regex captures the entire outermost
+ JSON block, including nested braces."""
+
+ # Create a minimal stub tokenizer that provides the few attributes the
+ # parser accesses (`version` and `get_vocab`).
+ class _StubMistralTokenizer(MistralTokenizer):
+ version = 11 # Satisfy the version check
+
+ def __init__(self):
+ pass
+
+ @staticmethod
+ def get_vocab():
+ # Provide the special TOOL_CALLS token expected by the parser.
+ return {"[TOOL_CALLS]": 0}
+
+ tokenizer = _StubMistralTokenizer()
+ parser = MistralToolParser(tokenizer)
+
+ # Craft a model output featuring nested JSON inside the arguments.
+ args_dict = {
+ "city": "Dallas",
+ "state": "TX",
+ "unit": "fahrenheit",
+ "sub_dict": {"foo": "bar", "inner": {"x": 1, "y": 2}},
+ }
+
+ model_output = f"{parser.bot_token}get_current_weather{json.dumps(args_dict)}"
+
+ parsed = parser.extract_tool_calls(model_output, None)
+
+ # Assertions: the tool call is detected and the full nested JSON is parsed
+ # without truncation.
+ assert parsed.tools_called
+
+ assert MistralToolCall.is_valid_id(parsed.tool_calls[0].id)
+ assert parsed.tool_calls[0].function.name == "get_current_weather"
+ assert json.loads(parsed.tool_calls[0].function.arguments) == args_dict
+ # No additional content outside the tool call should be returned.
+ assert parsed.content is None
+
+ # multiple calls
+ multiple_args_dict = [
+ {
+ "city": "Dallas",
+ "state": "TX",
+ "unit": "fahrenheit",
+ "sub_dict": {"foo": "bar", "inner": {"x": 1, "y": 2}},
+ },
+ {},
+ {"a": 0},
+ {"a": 1, "b": "c"},
+ ]
+ names = ["get_current_weather", "get_current_weather_2", "random", "random_2"]
+
+ model_output = "".join(
+ [
+ f"{parser.bot_token}{name}{json.dumps(args)}"
+ for name, args in zip(names, multiple_args_dict)
+ ]
+ )
+
+ parsed = parser.extract_tool_calls(model_output, None)
+
+ # Assertions: the tool call is detected and the full nested JSON is parsed
+ # without truncation.
+ assert parsed.tools_called
+ assert len(parsed.tool_calls) == len(multiple_args_dict)
+
+ for i, tool_call in enumerate(parsed.tool_calls):
+ assert MistralToolCall.is_valid_id(tool_call.id)
+ assert tool_call.function.name == names[i]
+ assert json.loads(tool_call.function.arguments) == multiple_args_dict[i]
+ # No additional content outside the tool call should be returned.
+ assert parsed.content is None
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation/test_phimoe.py b/asian_ai/performance/vLLM/tests/models/language/generation/test_phimoe.py
new file mode 100644
index 0000000000000000000000000000000000000000..0908f209437b88cc2c9e3753383a387ab3e9dc38
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/generation/test_phimoe.py
@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+import torch
+
+from vllm.platforms import current_platform
+
+from ....utils import large_gpu_test
+from ...utils import check_logprobs_close
+
+MODELS = [
+ "microsoft/Phi-3.5-MoE-instruct",
+]
+
+
+def test_phimoe_routing_function():
+ from vllm.model_executor.models.phimoe import phimoe_routing_function
+
+ test_case = {
+ 0: {
+ "hidden_states": torch.tensor(
+ [1, 2, 3, 4, 5, 6, 7, 8], dtype=torch.float32, requires_grad=False
+ ).view(4, 2),
+ "gating_output": torch.tensor(
+ [0.1, 0.2, 0.3, 0.4], dtype=torch.float32, requires_grad=False
+ ),
+ "topk": 2,
+ "renormalize": False,
+ },
+ 1: {
+ "hidden_states": torch.tensor(
+ [1, 2, 3, 4, 5, 6, 7, 8], dtype=torch.float32, requires_grad=False
+ ).view(4, 2),
+ "gating_output": torch.tensor(
+ [0.4, 0.2, 0.3, 0.4], dtype=torch.float32, requires_grad=False
+ ),
+ "topk": 2,
+ "renormalize": False,
+ },
+ }
+
+ ground_truth = {
+ 0: {
+ "topk_weights": torch.tensor(
+ [1.0, 1.0], dtype=torch.float32, requires_grad=False
+ ),
+ "topk_ids": torch.tensor([3, 2], dtype=torch.long, requires_grad=False),
+ },
+ 1: {
+ "topk_weights": torch.tensor(
+ [0.5, 1.0], dtype=torch.float32, requires_grad=False
+ ),
+ "topk_ids": torch.tensor([0, 3], dtype=torch.long, requires_grad=False),
+ },
+ }
+
+ for test_id in test_case:
+ topk_weights, topk_ids = phimoe_routing_function(**test_case[test_id])
+ assert torch.allclose(topk_weights, ground_truth[test_id]["topk_weights"])
+ assert torch.equal(topk_ids, ground_truth[test_id]["topk_ids"])
+
+
+# There is a known issue that triggers `AttributeError: 'DynamicCache'
+# object has no attribute 'seen_tokens'` when running:
+# `tests/models/language/generation/test_phimoe.py::test_models
+# [5-64-bfloat16-microsoft/Phi-3.5-MoE-instruct]`
+# This issue is being investigated and tracked in:
+# https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/discussions/58
+# It is platform-agnostic. Therefore, we skip this test on all platforms for now.
+@pytest.mark.skip(
+ reason="Skipping due to known issue: "
+ "'DynamicCache' object has no attribute 'seen_tokens'. See: "
+ "https://huggingface.co/microsoft/Phi-3.5-MoE-instruct/discussions/58 "
+ "for details.",
+)
+@pytest.mark.skipif(
+ condition=current_platform.is_cpu(),
+ reason="This test takes a lot time to run on CPU, "
+ "and vllm CI's disk space is not enough for this model.",
+)
+@large_gpu_test(min_gb=80)
+@pytest.mark.parametrize("model", MODELS)
+@pytest.mark.parametrize("dtype", ["bfloat16"])
+@pytest.mark.parametrize("max_tokens", [64])
+@pytest.mark.parametrize("num_logprobs", [5])
+def test_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+ max_tokens: int,
+ num_logprobs: int,
+) -> None:
+ with hf_runner(model, dtype=dtype) as hf_model:
+ hf_outputs = hf_model.generate_greedy_logprobs_limit(
+ example_prompts, max_tokens, num_logprobs
+ )
+
+ with vllm_runner(model, dtype=dtype) as vllm_model:
+ vllm_outputs = vllm_model.generate_greedy_logprobs(
+ example_prompts, max_tokens, num_logprobs
+ )
+ check_logprobs_close(
+ outputs_0_lst=hf_outputs,
+ outputs_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/__init__.py b/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/ppl_utils.py b/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/ppl_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..df97173e8e266d8d703ade8ba4ac0c03f28ff152
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/ppl_utils.py
@@ -0,0 +1,128 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# Adapted from https://huggingface.co/docs/transformers/perplexity
+from typing import cast
+
+import torch
+from datasets import load_dataset
+
+import tests.ci_envs as ci_envs
+from tests.models.utils import (
+ GenerateModelInfo,
+ TokensTextLogprobsPromptLogprobs,
+ get_vllm_extra_kwargs,
+)
+from vllm.logprobs import Logprob
+
+# See #24485
+PPL_TOL = 0.01
+MAX_LENGTH = 1024
+
+
+@torch.inference_mode
+def wikitext_ppl_test(
+ hf_runner,
+ vllm_runner,
+ model_info: GenerateModelInfo,
+ max_length=MAX_LENGTH,
+ vllm_extra_kwargs=None,
+ atol=PPL_TOL,
+):
+ vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
+
+ dataset = load_dataset("Salesforce/wikitext", "wikitext-2-raw-v1", split="test")
+
+ with vllm_runner(
+ model_info.name,
+ gpu_memory_utilization=0.7,
+ max_model_len=max_length,
+ max_num_seqs=1,
+ **vllm_extra_kwargs,
+ ) as vllm_model:
+ # Use max_num_seqs=1 to avoid OOM,
+ # and avoid batch different requests together.
+
+ model_config = vllm_model.llm.llm_engine.model_config
+
+ # Confirm whether vllm is using the correct architecture
+ if model_info.architecture:
+ assert model_info.architecture in model_config.architectures
+
+ max_length = min(model_config.max_model_len - 1, max_length)
+ stride = max_length
+
+ tokenizer = vllm_model.llm.get_tokenizer()
+ tokens = tokenizer.encode("\n\n".join(dataset["text"]))
+ n_tokens = len(tokens)
+
+ chunks = []
+ for begin_loc in range(0, n_tokens, stride):
+ end_loc = min(begin_loc + max_length, n_tokens)
+ chunks.append(tokens[begin_loc:end_loc])
+
+ outputs = vllm_model.generate_greedy_logprobs(
+ prompts=chunks,
+ max_tokens=1,
+ num_logprobs=None,
+ num_prompt_logprobs=0,
+ use_tqdm=False,
+ )
+ nll_sum = torch.tensor(0.0, dtype=torch.float32, device="cpu")
+ n_tokens = 0
+ for output in outputs:
+ output = cast(TokensTextLogprobsPromptLogprobs, output)
+ token_datas = cast(list[dict[int, Logprob] | None], output[3])
+
+ assert token_datas[0] is None
+ token_log_probs = []
+ for token_data in token_datas[1:]:
+ assert token_data is not None
+ assert len(token_data) == 1
+ token_log_prob = list(token_data.values())[0].logprob
+ token_log_probs.append(token_log_prob)
+
+ neg_log_likelihood = -torch.tensor(
+ token_log_probs, dtype=torch.float32, device="cpu"
+ ).sum()
+ nll_sum += neg_log_likelihood
+ n_tokens += len(token_log_probs)
+ vllm_ppl = float(torch.exp(nll_sum / n_tokens))
+ vllm_dtype = model_config.dtype
+ head_dtype = model_config.head_dtype
+
+ # Accelerate ppl test by setting Transformers ppl score to a constant
+ if model_info.hf_ppl is None:
+ with hf_runner(
+ model_info.name,
+ dtype=ci_envs.VLLM_CI_HF_DTYPE or model_info.hf_dtype,
+ ) as hf_model:
+ nll_sum = torch.tensor(0.0, dtype=torch.float32, device="cpu")
+ n_tokens = 0
+ for chunk in chunks:
+ inputs = hf_model.wrap_device({"input_ids": torch.tensor([chunk])})
+ input_ids = inputs["input_ids"]
+ outputs = hf_model.model(input_ids, labels=input_ids)
+ neg_log_likelihood = outputs.loss
+
+ neg_log_likelihood = neg_log_likelihood.to(torch.float32).cpu()
+
+ num_loss_tokens = len(chunk) - 1
+ nll_sum += neg_log_likelihood * num_loss_tokens
+ n_tokens += num_loss_tokens
+
+ hf_ppl = float(torch.exp(nll_sum / n_tokens))
+ hf_dtype = next(hf_model.model.parameters()).dtype
+ else:
+ hf_ppl = model_info.hf_ppl
+ hf_dtype = "Constant"
+
+ differ = (vllm_ppl - hf_ppl) / hf_ppl
+ print("Model:", model_info.name)
+ print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_ppl)
+ print("Transformers:", hf_dtype, hf_ppl)
+ print("Difference (%):", differ * 100)
+
+ # PPL the smaller, the better
+ # We are not concerned that the vllm PPL is less than Transformers,
+ # so we only perform one-sided testing.
+ assert differ < atol
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/test_gemma.py b/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/test_gemma.py
new file mode 100644
index 0000000000000000000000000000000000000000..a57a91b9d2a06a97b2ca488620bb90c1ee61601f
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/test_gemma.py
@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+
+from tests.models.utils import GenerateModelInfo
+
+from .ppl_utils import wikitext_ppl_test
+
+MODELS = [
+ GenerateModelInfo("google/gemma-2b", hf_ppl=21.48524284362793),
+ GenerateModelInfo("google/gemma-2-2b", hf_ppl=102.59290313720703),
+ GenerateModelInfo("google/gemma-3-4b-it", hf_ppl=27.79648208618164),
+]
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
+ wikitext_ppl_test(hf_runner, vllm_runner, model_info)
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/test_gpt.py b/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/test_gpt.py
new file mode 100644
index 0000000000000000000000000000000000000000..ddf0457b2918fe1f4151d29851c29247527c5871
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/test_gpt.py
@@ -0,0 +1,14 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+
+from tests.models.utils import GenerateModelInfo
+
+from .ppl_utils import wikitext_ppl_test
+
+MODELS = [GenerateModelInfo("openai-community/gpt2-large", hf_ppl=19.457056045532227)]
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
+ wikitext_ppl_test(hf_runner, vllm_runner, model_info)
diff --git a/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/test_qwen.py b/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/test_qwen.py
new file mode 100644
index 0000000000000000000000000000000000000000..c8bc0f134dd1e95fec8b9126865a83a059c657e9
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/generation_ppl_test/test_qwen.py
@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from tests.models.utils import GenerateModelInfo
+
+from .ppl_utils import wikitext_ppl_test
+
+MODELS = [
+ # for Qwen3
+ GenerateModelInfo("Qwen/Qwen3-0.6B", hf_ppl=23.864173889160156),
+ GenerateModelInfo("Qwen/Qwen3-0.6B-FP8", hf_ppl=24.313045501708984),
+ # for Qwen3.5
+ GenerateModelInfo("Qwen/Qwen3.5-0.8B", hf_ppl=19.38858413696289),
+]
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_ppl(hf_runner, vllm_runner, model_info: GenerateModelInfo):
+ vllm_extra_kwargs = {}
+ if model_info.name == "Qwen/Qwen3.5-0.8B":
+ vllm_extra_kwargs["language_model_only"] = True
+
+ wikitext_ppl_test(
+ hf_runner, vllm_runner, model_info, vllm_extra_kwargs=vllm_extra_kwargs
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/__init__.py b/asian_ai/performance/vLLM/tests/models/language/pooling/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/conftest.py b/asian_ai/performance/vLLM/tests/models/language/pooling/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..f8dabe1727d05b52c3444c70fd7777c90a8ba24b
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/conftest.py
@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Pytest configuration for vLLM language generation tests."""
+
+import warnings
+
+import torch
+
+from vllm.platforms import current_platform
+
+
+def pytest_sessionstart(session):
+ """Configure ROCm-specific settings before test session starts."""
+ if not current_platform.is_rocm():
+ return
+
+ # Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers
+ # accuracy issues: https://github.com/vllm-project/vllm/issues/30167
+ # TODO: Remove once ROCm SDP accuracy issues are resolved on HuggingFace
+ torch.backends.cuda.enable_flash_sdp(False)
+ torch.backends.cuda.enable_mem_efficient_sdp(False)
+ torch.backends.cuda.enable_math_sdp(True)
+ torch.set_float32_matmul_precision("high")
+ warnings.warn(
+ "ROCm: Disabled flash_sdp and mem_efficient_sdp, enabled math_sdp "
+ "to avoid HuggingFace Transformers accuracy issues",
+ UserWarning,
+ stacklevel=1,
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/embed_utils.py b/asian_ai/performance/vLLM/tests/models/language/pooling/embed_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..ae391574cc89ea75922f2a76a3e7fef403a37690
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/embed_utils.py
@@ -0,0 +1,81 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from collections.abc import Sequence
+
+import openai
+import pytest
+
+from tests.conftest import HfRunner
+from tests.models.utils import EmbedModelInfo, check_embeddings_close, matryoshka_fy
+
+
+def run_embedding_correctness_test(
+ hf_model: "HfRunner",
+ inputs: list[str],
+ vllm_outputs: Sequence[list[float]],
+ dimensions: int | None = None,
+):
+ hf_outputs = hf_model.encode(inputs)
+ if dimensions:
+ hf_outputs = matryoshka_fy(hf_outputs, dimensions)
+
+ check_embeddings_close(
+ embeddings_0_lst=hf_outputs,
+ embeddings_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ tol=1e-2,
+ )
+
+
+def correctness_test_embed_models(
+ hf_runner,
+ vllm_runner,
+ model_info: EmbedModelInfo,
+ example_prompts,
+ vllm_extra_kwargs=None,
+ hf_model_callback=None,
+):
+ pytest.skip("Debug only, ci prefers to use mteb test.")
+
+ # The example_prompts has ending "\n", for example:
+ # "Write a short story about a robot that dreams for the first time.\n"
+ # sentence_transformers will strip the input texts, see:
+ # https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
+ # This makes the input_ids different between hf_model and vllm_model.
+ # So we need to strip the input texts to avoid test failing.
+ example_prompts = [str(s).strip() for s in example_prompts]
+
+ vllm_extra_kwargs = vllm_extra_kwargs or {}
+ vllm_extra_kwargs["dtype"] = model_info.dtype
+
+ if model_info.hf_overrides is not None:
+ vllm_extra_kwargs["hf_overrides"] = model_info.hf_overrides
+
+ with vllm_runner(
+ model_info.name, runner="pooling", max_model_len=None, **vllm_extra_kwargs
+ ) as vllm_model:
+ vllm_outputs = vllm_model.embed(example_prompts)
+
+ with hf_runner(
+ model_info.name,
+ dtype=model_info.hf_dtype,
+ is_sentence_transformer=True,
+ ) as hf_model:
+ if hf_model_callback is not None:
+ hf_model_callback(hf_model)
+
+ run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
+
+
+async def run_client_embeddings(
+ client: openai.AsyncOpenAI,
+ model_name: str,
+ queries: list[str],
+ instruction: str = "",
+) -> list[list[float]]:
+ outputs = await client.embeddings.create(
+ model=model_name,
+ input=[instruction + q for q in queries],
+ )
+ return [data.embedding for data in outputs.data]
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py
new file mode 100644
index 0000000000000000000000000000000000000000..a99e2e306e993026b699ba7356f8d56916a728d0
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_all_pooling_plus_chunked_prefill.py
@@ -0,0 +1,55 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+import torch
+from transformers import AutoModel
+
+from tests.models.utils import check_embeddings_close
+from vllm import TokensPrompt
+from vllm.config import PoolerConfig
+
+
+@pytest.mark.parametrize(
+ "model",
+ ["Qwen/Qwen3-Embedding-0.6B"],
+)
+@torch.inference_mode
+def test_embed_models(hf_runner, vllm_runner, model: str):
+ chunk_size = 10
+ n_prompt_tokens = [55, 56, 57]
+ token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
+
+ with vllm_runner(
+ model,
+ runner="pooling",
+ pooler_config=PoolerConfig(task="token_embed"),
+ max_model_len=128,
+ max_num_batched_tokens=chunk_size,
+ enforce_eager=True,
+ # `enable_chunked_prefill`: Set to `False` instead of `None` in VllmRunner
+ enable_chunked_prefill=True,
+ enable_prefix_caching=True,
+ ) as vllm_model:
+ vllm_outputs = vllm_model.token_embed(
+ [TokensPrompt(prompt_token_ids=t) for t in token_prompts],
+ )
+
+ with hf_runner(
+ model,
+ auto_cls=AutoModel,
+ ) as hf_model:
+ hf_outputs = []
+ for token_prompt in token_prompts:
+ inputs = hf_model.wrap_device({"input_ids": torch.tensor([token_prompt])})
+ input_ids = inputs["input_ids"]
+ output = hf_model.model(input_ids)
+ hf_outputs.append(output.last_hidden_state.cpu().float()[0])
+
+ for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+ check_embeddings_close(
+ embeddings_0_lst=hf_output,
+ embeddings_1_lst=vllm_output,
+ name_0="hf",
+ name_1="vllm",
+ tol=1e-2,
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_auto_prefix_cache_support.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_auto_prefix_cache_support.py
new file mode 100644
index 0000000000000000000000000000000000000000..1b4d681299c497165ba4728942f63b40e12bcfd1
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_auto_prefix_cache_support.py
@@ -0,0 +1,113 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+import torch
+from transformers import AutoModelForSequenceClassification
+
+from tests.models.language.pooling.embed_utils import run_embedding_correctness_test
+
+
+@pytest.mark.parametrize(
+ "model",
+ ["jason9693/Qwen2.5-1.5B-apeach"],
+)
+@pytest.mark.parametrize("dtype", ["half"])
+def test_classify_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ # example_prompts is too short for testing prefix_caching
+ example_prompts = [s * 10 for s in example_prompts]
+
+ with vllm_runner(
+ model, max_model_len=512, dtype=dtype, enable_prefix_caching=True
+ ) as vllm_model:
+ vllm_config = vllm_model.llm.llm_engine.vllm_config
+ cache_config = vllm_config.cache_config
+ assert cache_config.enable_prefix_caching
+
+ # First Run
+ vllm_model.classify(example_prompts)
+
+ # assert prefix_caching works
+ pooling_outputs = vllm_model.llm.encode(
+ example_prompts, pooling_task="classify"
+ )
+ for output in pooling_outputs:
+ assert output.num_cached_tokens > 0
+ vllm_outputs = [req_output.outputs.data for req_output in pooling_outputs]
+
+ with hf_runner(
+ model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
+ ) as hf_model:
+ hf_outputs = hf_model.classify(example_prompts)
+
+ for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+ hf_output = torch.tensor(hf_output)
+ vllm_output = torch.tensor(vllm_output)
+
+ assert torch.allclose(
+ hf_output, vllm_output, 1e-3 if dtype == "float" else 1e-2
+ )
+
+
+@pytest.mark.parametrize(
+ "model",
+ ["Qwen/Qwen3-Embedding-0.6B"],
+)
+@pytest.mark.parametrize("dtype", ["half"])
+def test_embed_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+):
+ # example_prompts is too short for testing prefix_caching
+ example_prompts = [str(s).strip() * 10 for s in example_prompts]
+
+ with vllm_runner(
+ model,
+ runner="pooling",
+ max_model_len=None,
+ enable_prefix_caching=True,
+ ) as vllm_model:
+ vllm_config = vllm_model.llm.llm_engine.vllm_config
+ cache_config = vllm_config.cache_config
+ assert cache_config.enable_prefix_caching
+
+ # First Run
+ vllm_model.embed(example_prompts)
+
+ # assert prefix_caching works
+ pooling_outputs = vllm_model.llm.encode(example_prompts, pooling_task="embed")
+ for output in pooling_outputs:
+ assert output.num_cached_tokens > 0
+ vllm_outputs = [req_output.outputs.data for req_output in pooling_outputs]
+
+ with hf_runner(
+ model,
+ is_sentence_transformer=True,
+ ) as hf_model:
+ run_embedding_correctness_test(hf_model, example_prompts, vllm_outputs)
+
+
+@pytest.mark.parametrize(
+ "model",
+ [
+ "intfloat/e5-small",
+ "Alibaba-NLP/gte-Qwen2-1.5B-instruct", # is_causal == False
+ "papluca/xlm-roberta-base-language-detection",
+ ],
+)
+@pytest.mark.parametrize("dtype", ["half"])
+def test_non_causal_models(
+ hf_runner, vllm_runner, example_prompts, model: str, dtype: str
+) -> None:
+ with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
+ vllm_config = vllm_model.llm.llm_engine.vllm_config
+ cache_config = vllm_config.cache_config
+ assert not cache_config.enable_prefix_caching
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_bge_m3.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_bge_m3.py
new file mode 100644
index 0000000000000000000000000000000000000000..c00b8c308098b044d64f91711905c73eec486c3e
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_bge_m3.py
@@ -0,0 +1,212 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import httpx
+import openai
+import pytest
+import torch
+
+from ....utils import RemoteOpenAIServer
+from .embed_utils import run_client_embeddings
+
+MODEL_NAME = "BAAI/bge-m3"
+MAX_MODEL_LEN = 512
+
+
+# Example from https://huggingface.co/BAAI/bge-m3
+sentences_1 = ["What is BGE M3?", "Definition of BM25"]
+sentences_2 = [
+ "BGE M3 is an embedding model supporting dense retrieval, "
+ "lexical matching and multi-vector interaction.",
+ "BM25 is a bag-of-words retrieval function that ranks a set "
+ "of documents based on the query terms appearing in each document",
+]
+
+similarity_reference = [[0.6259, 0.3474], [0.3309, 0.6734]]
+lexical_score_reference = [0.19554901123046875, 0.0]
+colbert_score_reference = [0.7797, 0.4620]
+SUPPORTED_TASKS = ["embed", "token_embed", "token_classify"]
+
+
+@pytest.fixture(scope="module", params=SUPPORTED_TASKS)
+def pooling_task(request):
+ yield request.param
+
+
+@pytest.fixture(scope="module")
+def server(pooling_task):
+ args = [
+ "--max-model-len",
+ str(MAX_MODEL_LEN),
+ "--hf-overrides",
+ '{"architectures": ["BgeM3EmbeddingModel"]}',
+ "--pooler-config.task",
+ pooling_task,
+ ]
+
+ with RemoteOpenAIServer(MODEL_NAME, args) as remote_server:
+ yield remote_server
+
+
+@pytest.mark.asyncio
+async def test_bge_m3_api_server_embedding(server, pooling_task):
+ client = server.get_async_client()
+
+ if pooling_task != "embed":
+ with pytest.raises(openai.InternalServerError):
+ await run_client_embeddings(
+ client,
+ MODEL_NAME,
+ sentences_1,
+ )
+ return
+
+ embeddings_list_1 = await run_client_embeddings(
+ client,
+ MODEL_NAME,
+ sentences_1,
+ )
+ embeddings_list_2 = await run_client_embeddings(
+ client,
+ MODEL_NAME,
+ sentences_2,
+ )
+
+ embeddings_1 = torch.tensor(embeddings_list_1)
+ embeddings_2 = torch.tensor(embeddings_list_2)
+ similarity = embeddings_1 @ embeddings_2.T
+
+ # reference values from BAAI/bge-m3 documentation
+ reference = torch.tensor(similarity_reference)
+
+ assert torch.allclose(similarity, reference, rtol=0.01)
+
+
+async def tokenize(client: openai.AsyncOpenAI, sentences: list[str]) -> list[list[int]]:
+ futures = []
+ for sentence in sentences:
+ futures.append(
+ client.post(
+ "../tokenize",
+ body={"model": MODEL_NAME, "prompt": sentence},
+ cast_to=httpx.Response,
+ )
+ )
+ return [(await future).json()["tokens"] for future in futures]
+
+
+async def sparse_embeddings(
+ client: openai.AsyncOpenAI, sentences: list[str]
+) -> list[dict[int, float]]:
+ all_tokens = await tokenize(client, sentences)
+ result = await client.post(
+ "../pooling",
+ body={"model": MODEL_NAME, "input": sentences, "task": "token_classify"},
+ cast_to=httpx.Response,
+ )
+ all_embeddings = [data["data"] for data in result.json()["data"]]
+
+ ret = []
+
+ for sent_tokens, sent_emb in zip(all_tokens, all_embeddings):
+ token_embs = dict[int, float]()
+ if sent_tokens[0] == 0:
+ sent_tokens = sent_tokens[1:]
+ for token, val in zip(sent_tokens, sent_emb):
+ token_embs[token] = max(val, token_embs.get(token, 0.0))
+ ret.append(token_embs)
+ return ret
+
+
+# Based on https://github.com/FlagOpen/FlagEmbedding/blob/6fd176266f2382878bcc69cd656cff425d52f49b/FlagEmbedding/inference/embedder/encoder_only/m3.py#L129
+def compute_lexical_matching_score(
+ lw1: dict[int, float], lw2: dict[int, float]
+) -> float:
+ scores = 0.0
+ for token, weight in lw1.items():
+ if token in lw2:
+ scores += weight * lw2[token]
+ return scores
+
+
+@pytest.mark.asyncio
+async def test_bge_m3_api_server_sparse_embedding(server, pooling_task):
+ client = server.get_async_client()
+
+ if pooling_task != "token_classify":
+ with pytest.raises(openai.BadRequestError):
+ await sparse_embeddings(client, sentences_1)
+ return
+
+ embeddings_1 = await sparse_embeddings(client, sentences_1)
+ embeddings_2 = await sparse_embeddings(client, sentences_2)
+
+ lexical_scores_1_0_x_2_0 = compute_lexical_matching_score(
+ embeddings_1[0], embeddings_2[0]
+ )
+ assert lexical_scores_1_0_x_2_0 == pytest.approx(
+ lexical_score_reference[0], rel=0.01
+ )
+
+ lexical_scores_1_0_x_1_1 = compute_lexical_matching_score(
+ embeddings_1[0], embeddings_1[1]
+ )
+ assert lexical_scores_1_0_x_1_1 == pytest.approx(
+ lexical_score_reference[1], rel=0.01
+ )
+
+
+@pytest.mark.asyncio
+async def test_bge_m3_api_server_sparse_embedding_corner_case(server, pooling_task):
+ if pooling_task != "token_classify":
+ return
+
+ client = server.get_async_client()
+ embeddings = await sparse_embeddings(client, ["Hi"])
+ assert len(embeddings) == 1
+ assert 2673 in embeddings[0]
+ assert embeddings[0][2673] == pytest.approx(0.26710861921310425, rel=0.01)
+
+
+# https://github.com/FlagOpen/FlagEmbedding/blob/6fd176266f2382878bcc69cd656cff425d52f49b/FlagEmbedding/inference/embedder/encoder_only/m3.py#L163
+def colbert_score(q_reps: torch.Tensor, p_reps: torch.Tensor) -> torch.Tensor:
+ token_scores = torch.einsum("in,jn->ij", q_reps, p_reps)
+ scores, _ = token_scores.max(-1)
+ scores = torch.sum(scores) / q_reps.size(0)
+ return scores
+
+
+@pytest.mark.asyncio
+async def test_bge_m3_api_server_multi_vector(server, pooling_task):
+ client = server.get_async_client()
+
+ if pooling_task != "token_embed":
+ with pytest.raises(openai.BadRequestError):
+ await client.post(
+ "../pooling",
+ body={"model": MODEL_NAME, "input": sentences_1, "task": "token_embed"},
+ cast_to=httpx.Response,
+ )
+ return
+
+ result_1 = await client.post(
+ "../pooling",
+ body={"model": MODEL_NAME, "input": sentences_1, "task": "token_embed"},
+ cast_to=httpx.Response,
+ )
+ embeddings_1 = [torch.tensor(data["data"]) for data in result_1.json()["data"]]
+
+ result_2 = await client.post(
+ "../pooling",
+ body={"model": MODEL_NAME, "input": sentences_2, "task": "token_embed"},
+ cast_to=httpx.Response,
+ )
+ embeddings_2 = [torch.tensor(data["data"]) for data in result_2.json()["data"]]
+
+ colbert_score_1_0_x_2_0 = colbert_score(embeddings_1[0], embeddings_2[0])
+ assert colbert_score_1_0_x_2_0 == pytest.approx(
+ colbert_score_reference[0], rel=0.01
+ )
+ colbert_score_1_0_x_2_1 = colbert_score(embeddings_1[0], embeddings_2[1])
+ assert colbert_score_1_0_x_2_1 == pytest.approx(
+ colbert_score_reference[1], rel=0.01
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_classification.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_classification.py
new file mode 100644
index 0000000000000000000000000000000000000000..526d6e301366fc106013e571ef7a2ab9413683a6
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_classification.py
@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+import torch
+from transformers import AutoModelForSequenceClassification
+
+from vllm.platforms import current_platform
+
+
+@pytest.mark.parametrize(
+ "model",
+ [
+ pytest.param(
+ "jason9693/Qwen2.5-1.5B-apeach",
+ marks=[
+ pytest.mark.core_model,
+ pytest.mark.cpu_model,
+ pytest.mark.slow_test,
+ ],
+ ),
+ ],
+)
+@pytest.mark.parametrize("dtype", ["half"] if current_platform.is_rocm() else ["float"])
+def test_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
+ vllm_outputs = vllm_model.classify(example_prompts)
+
+ with hf_runner(
+ model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
+ ) as hf_model:
+ hf_outputs = hf_model.classify(example_prompts)
+
+ # check logits difference
+ for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+ hf_output = torch.tensor(hf_output)
+ vllm_output = torch.tensor(vllm_output)
+
+ # the tolerance value of 1e-2 is selected based on the
+ # half datatype tests in
+ # tests/models/language/pooling/test_embedding.py
+ assert torch.allclose(
+ hf_output,
+ vllm_output,
+ rtol=2e-3 if dtype == "float" else 1e-2,
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_colbert.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_colbert.py
new file mode 100644
index 0000000000000000000000000000000000000000..2449da58828722260233d37d6c88a09b0cf81b93
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_colbert.py
@@ -0,0 +1,420 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Tests for ColBERT late interaction scoring.
+
+Tests are parametrized across multiple ColBERT backbones to ensure the
+generic ColBERT support works with different encoder architectures.
+"""
+
+from contextlib import contextmanager
+
+import pytest
+import torch
+
+from tests.utils import wait_for_rocm_memory_to_settle
+from vllm.distributed import cleanup_dist_env_and_memory
+from vllm.entrypoints.pooling.scoring.utils import compute_maxsim_score
+
+# -----------------------------------------------------------------------
+# Model definitions: (model_name, colbert_dim, extra vllm_runner kwargs)
+# -----------------------------------------------------------------------
+COLBERT_MODELS = {
+ "bert": {
+ "model": "answerdotai/answerai-colbert-small-v1",
+ "colbert_dim": 96,
+ "max_model_len": 512,
+ "extra_kwargs": {},
+ "hf_comparison": {
+ "weights_file": "model.safetensors",
+ "weights_key": "linear.weight",
+ "trust_remote_code": False,
+ "model_cls": "BertModel",
+ },
+ },
+ "modernbert": {
+ "model": "lightonai/GTE-ModernColBERT-v1",
+ "colbert_dim": 128,
+ "max_model_len": 299,
+ "extra_kwargs": {
+ "hf_overrides": {
+ "architectures": ["ColBERTModernBertModel"],
+ },
+ },
+ "hf_comparison": {
+ "weights_file": "1_Dense/model.safetensors",
+ "weights_key": "linear.weight",
+ "trust_remote_code": False,
+ "model_cls": "AutoModel",
+ },
+ },
+ "jina": {
+ "model": "jinaai/jina-colbert-v2",
+ "colbert_dim": 128,
+ "max_model_len": 8192,
+ "extra_kwargs": {
+ "hf_overrides": {
+ "architectures": ["ColBERTJinaRobertaModel"],
+ },
+ },
+ "hf_comparison": {
+ "weights_file": "model.safetensors",
+ "weights_key": "linear.weight",
+ "trust_remote_code": True,
+ "model_cls": "AutoModel",
+ },
+ },
+ "lfm2": {
+ "model": "LiquidAI/LFM2-ColBERT-350M",
+ "colbert_dim": 128,
+ "max_model_len": 511,
+ "extra_kwargs": {
+ "hf_overrides": {
+ "architectures": ["ColBERTLfm2Model"],
+ },
+ },
+ "hf_comparison": {
+ "weights_file": "1_Dense/model.safetensors",
+ "weights_key": "linear.weight",
+ "trust_remote_code": False,
+ "model_cls": "AutoModel",
+ },
+ },
+}
+
+
+TEXTS_1 = [
+ "What is the capital of France?",
+ "What is the capital of Germany?",
+]
+
+TEXTS_2 = [
+ "The capital of France is Paris.",
+ "The capital of Germany is Berlin.",
+]
+
+DTYPE = "half"
+
+
+def _load_hf_model(model_name: str, hf_spec: dict, device: torch.device):
+ """Load HF model on the given device with a compatible attention impl."""
+ from transformers import AutoModel, BertModel
+
+ cls = BertModel if hf_spec["model_cls"] == "BertModel" else AutoModel
+ trust = hf_spec.get("trust_remote_code", False)
+
+ # Flash / Triton kernels require GPU tensors; fall back to eager on CPU.
+ extra = {}
+ if device.type == "cpu":
+ extra["attn_implementation"] = "eager"
+
+ model = cls.from_pretrained(
+ model_name,
+ trust_remote_code=trust,
+ **extra,
+ ).to(device)
+ model.eval()
+
+ # Transformers 5.0 weight materialization can clear non-persistent
+ # buffers (e.g. rotary inv_freq) that were registered with
+ # persistent=False. Re-compute them so the model produces valid output.
+ for mod in model.modules():
+ if hasattr(mod, "_compute_inv_freq") and hasattr(mod, "inv_freq"):
+ mod.inv_freq = mod._compute_inv_freq(device=device)
+
+ return model
+
+
+def _load_projection_weight(model_name: str, hf_spec: dict, device: torch.device):
+ """Download and return the ColBERT linear projection weight."""
+ from huggingface_hub import hf_hub_download
+ from safetensors.torch import load_file
+
+ path = hf_hub_download(model_name, filename=hf_spec["weights_file"])
+ weights = load_file(path)
+ return weights[hf_spec["weights_key"]].to(device)
+
+
+def _compute_hf_colbert_embeddings(model, tokenizer, linear_weight, texts, device):
+ """Run HF model + projection and return L2-normalised token embeddings."""
+ import torch.nn.functional as F
+
+ embeddings = []
+ for text in texts:
+ inputs = tokenizer(text, return_tensors="pt").to(device)
+ with torch.no_grad():
+ hidden = model(**inputs).last_hidden_state.float()
+ projected = F.linear(hidden, linear_weight.float())
+ normalised = F.normalize(projected, p=2, dim=-1)
+ embeddings.append(normalised.squeeze(0).cpu())
+ return embeddings
+
+
+@contextmanager
+def _hf_colbert_model(model_name: str, hf_spec: dict, device: torch.device):
+ """Load the HF backbone + ColBERT projection, freeing the GPU on exit.
+
+ These live outside any runner context, so without explicit cleanup ROCm
+ keeps the VRAM resident and the next backend parametrization (or test)
+ OOMs on startup.
+ """
+ hf_model = _load_hf_model(model_name, hf_spec, device)
+ linear_weight = _load_projection_weight(model_name, hf_spec, device)
+ try:
+ yield hf_model, linear_weight
+ finally:
+ del hf_model, linear_weight
+ cleanup_dist_env_and_memory()
+ wait_for_rocm_memory_to_settle()
+
+
+def _assert_embeddings_close(vllm_outputs, hf_embeddings):
+ """Assert that vLLM and HuggingFace embeddings match."""
+ for i, (hf_emb, vllm_out) in enumerate(zip(hf_embeddings, vllm_outputs)):
+ vllm_emb = torch.as_tensor(vllm_out).float()
+
+ assert hf_emb.shape == vllm_emb.shape, (
+ f"Shape mismatch for text {i}: HF {hf_emb.shape} vs vLLM {vllm_emb.shape}"
+ )
+
+ torch.testing.assert_close(
+ vllm_emb,
+ hf_emb,
+ rtol=1e-2,
+ atol=1e-2,
+ msg=f"Embedding mismatch for text {i}",
+ )
+
+
+@pytest.fixture(params=list(COLBERT_MODELS.keys()), scope="module")
+def colbert_spec(request):
+ """Return the model spec dict for the current parametrization."""
+ return COLBERT_MODELS[request.param]
+
+
+@pytest.fixture(scope="module")
+def colbert_model_name(colbert_spec):
+ return colbert_spec["model"]
+
+
+@pytest.fixture(scope="module")
+def colbert_dim(colbert_spec):
+ return colbert_spec["colbert_dim"]
+
+
+@pytest.fixture(scope="module")
+def colbert_max_model_len(colbert_spec):
+ return colbert_spec["max_model_len"]
+
+
+@pytest.fixture(scope="module")
+def colbert_extra_kwargs(colbert_spec):
+ return colbert_spec["extra_kwargs"]
+
+
+def test_colbert_token_embed(
+ vllm_runner,
+ colbert_model_name,
+ colbert_dim,
+ colbert_max_model_len,
+ colbert_extra_kwargs,
+):
+ """Test that ColBERT model produces token embeddings."""
+ with vllm_runner(
+ colbert_model_name,
+ runner="pooling",
+ dtype=DTYPE,
+ max_model_len=colbert_max_model_len,
+ enforce_eager=True,
+ **colbert_extra_kwargs,
+ ) as vllm_model:
+ outputs = vllm_model.token_embed([TEXTS_1[0]])
+
+ assert len(outputs) == 1
+ emb = torch.as_tensor(outputs[0])
+ assert emb.dim() == 2
+ assert emb.shape[1] == colbert_dim
+ assert emb.shape[0] > 1
+
+
+def test_colbert_late_interaction_1_to_1(
+ vllm_runner,
+ colbert_model_name,
+ colbert_max_model_len,
+ colbert_extra_kwargs,
+):
+ """Test ColBERT late interaction scoring with 1:1 query-document pair."""
+ with vllm_runner(
+ colbert_model_name,
+ runner="pooling",
+ dtype=DTYPE,
+ max_model_len=colbert_max_model_len,
+ enforce_eager=True,
+ **colbert_extra_kwargs,
+ ) as vllm_model:
+ q_outputs = vllm_model.token_embed([TEXTS_1[0]])
+ d_outputs = vllm_model.token_embed([TEXTS_2[0]])
+
+ q_emb = torch.as_tensor(q_outputs[0])
+ d_emb = torch.as_tensor(d_outputs[0])
+
+ manual_score = compute_maxsim_score(q_emb, d_emb).item()
+
+ vllm_scores = vllm_model.score(TEXTS_1[0], TEXTS_2[0])
+
+ assert len(vllm_scores) == 1
+ assert vllm_scores[0] == pytest.approx(manual_score, rel=0.01)
+
+
+def test_colbert_late_interaction_1_to_N(
+ vllm_runner,
+ colbert_model_name,
+ colbert_max_model_len,
+ colbert_extra_kwargs,
+):
+ """Test ColBERT late interaction scoring with 1:N query-documents."""
+ with vllm_runner(
+ colbert_model_name,
+ runner="pooling",
+ dtype=DTYPE,
+ max_model_len=colbert_max_model_len,
+ enforce_eager=True,
+ **colbert_extra_kwargs,
+ ) as vllm_model:
+ q_outputs = vllm_model.token_embed([TEXTS_1[0]])
+ d_outputs = vllm_model.token_embed(TEXTS_2)
+
+ q_emb = torch.as_tensor(q_outputs[0])
+
+ manual_scores = []
+ for d_out in d_outputs:
+ d_emb = torch.as_tensor(d_out)
+ manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
+
+ vllm_scores = vllm_model.score(TEXTS_1[0], TEXTS_2)
+
+ assert len(vllm_scores) == 2
+ for i in range(2):
+ assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
+
+
+def test_colbert_late_interaction_N_to_N(
+ vllm_runner,
+ colbert_model_name,
+ colbert_max_model_len,
+ colbert_extra_kwargs,
+):
+ """Test ColBERT late interaction scoring with N:N query-documents."""
+ with vllm_runner(
+ colbert_model_name,
+ runner="pooling",
+ dtype=DTYPE,
+ max_model_len=colbert_max_model_len,
+ enforce_eager=True,
+ **colbert_extra_kwargs,
+ ) as vllm_model:
+ q_outputs = vllm_model.token_embed(TEXTS_1)
+ d_outputs = vllm_model.token_embed(TEXTS_2)
+
+ manual_scores = []
+ for q_out, d_out in zip(q_outputs, d_outputs):
+ q_emb = torch.as_tensor(q_out)
+ d_emb = torch.as_tensor(d_out)
+ manual_scores.append(compute_maxsim_score(q_emb, d_emb).item())
+
+ vllm_scores = vllm_model.score(TEXTS_1, TEXTS_2)
+
+ assert len(vllm_scores) == 2
+ for i in range(2):
+ assert vllm_scores[i] == pytest.approx(manual_scores[i], rel=0.01)
+
+
+def test_colbert_relevance_ordering(
+ vllm_runner,
+ colbert_model_name,
+ colbert_max_model_len,
+ colbert_extra_kwargs,
+):
+ """Test that ColBERT scores relevant documents higher than irrelevant."""
+ query = "What is machine learning?"
+ documents = [
+ "Machine learning is a subset of artificial intelligence.",
+ "Python is a programming language.",
+ "Deep learning uses neural networks.",
+ ]
+
+ with vllm_runner(
+ colbert_model_name,
+ runner="pooling",
+ dtype=DTYPE,
+ max_model_len=colbert_max_model_len,
+ enforce_eager=True,
+ **colbert_extra_kwargs,
+ ) as vllm_model:
+ scores = vllm_model.score(query, documents)
+
+ assert len(scores) == 3
+ assert scores[0] > scores[1], "ML doc should score higher than Python doc"
+ assert scores[2] > scores[1], "DL doc should score higher than Python doc"
+
+
+def test_colbert_embed_not_supported(
+ vllm_runner,
+ colbert_model_name,
+ colbert_max_model_len,
+ colbert_extra_kwargs,
+):
+ """Test that ColBERT model does not support 'embed' task."""
+ with (
+ vllm_runner(
+ colbert_model_name,
+ runner="pooling",
+ dtype=DTYPE,
+ max_model_len=colbert_max_model_len,
+ enforce_eager=True,
+ **colbert_extra_kwargs,
+ ) as vllm_model,
+ pytest.raises(ValueError, match="Embedding API is not supported"),
+ ):
+ vllm_model.embed([TEXTS_1[0]])
+
+
+@pytest.mark.parametrize("backend", list(COLBERT_MODELS.keys()))
+def test_colbert_hf_comparison(vllm_runner, backend):
+ """Test that vLLM ColBERT embeddings match HuggingFace for each backend."""
+ from transformers import AutoTokenizer
+
+ spec = COLBERT_MODELS[backend]
+ hf_spec = spec["hf_comparison"]
+ extra_kwargs = spec["extra_kwargs"]
+ model_name = spec["model"]
+ assert isinstance(model_name, str)
+ assert isinstance(hf_spec, dict)
+ assert isinstance(extra_kwargs, dict)
+ test_texts = [TEXTS_1[0], TEXTS_2[0]]
+
+ with vllm_runner(
+ model_name,
+ runner="pooling",
+ dtype="float32",
+ max_model_len=spec["max_model_len"],
+ enforce_eager=True,
+ **extra_kwargs,
+ ) as vllm_model:
+ vllm_outputs = vllm_model.token_embed(test_texts)
+
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+
+ hf_tokenizer = AutoTokenizer.from_pretrained(
+ model_name,
+ trust_remote_code=hf_spec.get("trust_remote_code", False),
+ )
+ with _hf_colbert_model(model_name, hf_spec, device) as (hf_model, linear_weight):
+ hf_embeddings = _compute_hf_colbert_embeddings(
+ hf_model,
+ hf_tokenizer,
+ linear_weight,
+ test_texts,
+ device,
+ )
+
+ _assert_embeddings_close(vllm_outputs, hf_embeddings)
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_embedding.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_embedding.py
new file mode 100644
index 0000000000000000000000000000000000000000..42ec18ec7ce8a219c84303aee6e3307982c8ad63
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_embedding.py
@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from vllm.config import PoolerConfig
+
+from ...utils import check_embeddings_close
+
+
+@pytest.mark.parametrize(
+ "model",
+ [
+ # Be careful of the order of models, decoder-only models should be
+ # placed before encoder-only models, otherwise `Qwen2.5-0.5B-Instruct`
+ # case won't pass because gte-Qwen2-1.5B-instruct will cache custom
+ # model code with bidirectional attention.
+ # [Decoder-only]
+ pytest.param(
+ "BAAI/bge-multilingual-gemma2",
+ marks=[pytest.mark.core_model, pytest.mark.slow_test],
+ ),
+ pytest.param(
+ "intfloat/e5-mistral-7b-instruct",
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
+ ),
+ pytest.param(
+ "ssmits/Qwen2-7B-Instruct-embed-base", marks=[pytest.mark.cpu_model]
+ ),
+ # [Encoder-only]
+ pytest.param(
+ "BAAI/bge-base-en-v1.5",
+ marks=[
+ pytest.mark.core_model,
+ pytest.mark.cpu_model,
+ pytest.mark.slow_test,
+ ],
+ ),
+ pytest.param("sentence-transformers/all-MiniLM-L12-v2"),
+ pytest.param("intfloat/multilingual-e5-small"),
+ # [Cross-Encoder]
+ pytest.param(
+ "sentence-transformers/stsb-roberta-base-v2",
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
+ ),
+ ],
+)
+def test_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model,
+) -> None:
+ vllm_extra_kwargs = {}
+ if model == "ssmits/Qwen2-7B-Instruct-embed-base":
+ vllm_extra_kwargs["pooler_config"] = PoolerConfig(
+ seq_pooling_type="MEAN", use_activation=False
+ )
+
+ max_model_len: int | None = 512
+ if model in [
+ "sentence-transformers/all-MiniLM-L12-v2",
+ "sentence-transformers/stsb-roberta-base-v2",
+ ]:
+ max_model_len = None
+
+ # The example_prompts has ending "\n", for example:
+ # "Write a short story about a robot that dreams for the first time.\n"
+ # sentence_transformers will strip the input texts, see:
+ # https://github.com/UKPLab/sentence-transformers/blob/v3.1.1/sentence_transformers/models/Transformer.py#L159
+ # This makes the input_ids different between hf_model and vllm_model.
+ # So we need to strip the input texts to avoid test failing.
+ example_prompts = [str(s).strip() for s in example_prompts]
+
+ with hf_runner(model, is_sentence_transformer=True) as hf_model:
+ hf_outputs = hf_model.encode(example_prompts)
+
+ with vllm_runner(
+ model, runner="pooling", max_model_len=max_model_len, **vllm_extra_kwargs
+ ) as vllm_model:
+ vllm_outputs = vllm_model.embed(example_prompts)
+
+ check_embeddings_close(
+ embeddings_0_lst=hf_outputs,
+ embeddings_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ tol=1e-2,
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_extract_hidden_states.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_extract_hidden_states.py
new file mode 100644
index 0000000000000000000000000000000000000000..eb1d53cbf738c1a1bb839f104aae6f8d8e5942da
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_extract_hidden_states.py
@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+import torch
+
+from vllm import TokensPrompt
+from vllm.config import PoolerConfig
+
+
+@pytest.mark.parametrize(
+ "model",
+ ["Qwen/Qwen3-0.6B"],
+)
+@torch.inference_mode
+def test_extract_hidden_states(hf_runner, vllm_runner, model: str):
+ n_prompt_tokens = [55, 56, 57]
+ token_prompts = [[1024 + i for i in range(n)] for n in n_prompt_tokens]
+
+ with vllm_runner(
+ model,
+ max_model_len=128,
+ enforce_eager=True,
+ runner="pooling",
+ pooler_config=PoolerConfig(task="token_embed"),
+ enable_prefix_caching=True,
+ ) as vllm_model:
+ pooling_outputs = vllm_model.llm.encode(
+ [TokensPrompt(prompt_token_ids=t) for t in token_prompts],
+ pooling_task="token_embed",
+ )
+
+ for n, output in zip(n_prompt_tokens, pooling_outputs):
+ assert len(output.prompt_token_ids) == n
+ assert len(output.outputs.data) == n
+ assert output.num_cached_tokens == 0
+
+ # test enable_prefix_caching plus all pooling
+ # we need to skip reading cache at this request by
+ # request.skip_reading_prefix_cache
+ pooling_outputs = vllm_model.llm.encode(
+ [TokensPrompt(prompt_token_ids=t) for t in token_prompts],
+ pooling_task="token_embed",
+ )
+
+ for n, output in zip(n_prompt_tokens, pooling_outputs):
+ assert len(output.prompt_token_ids) == n
+ assert len(output.outputs.data) == n
+ assert output.num_cached_tokens == 0
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_gritlm.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_gritlm.py
new file mode 100644
index 0000000000000000000000000000000000000000..bae170cdb6eedbcddf051fc4a936f6d1ba30bea9
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_gritlm.py
@@ -0,0 +1,188 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import numpy as np
+import pytest
+from scipy.spatial.distance import cosine
+
+from vllm import LLM, SamplingParams
+from vllm.config import ModelConfig
+
+from ....utils import ROCM_ENV_OVERRIDES, ROCM_EXTRA_ARGS, RemoteOpenAIServer
+from .embed_utils import run_client_embeddings
+
+MODEL_NAME = "parasail-ai/GritLM-7B-vllm"
+MAX_MODEL_LEN = 4000
+ATOL = 2.3e-3
+
+
+def _arr(arr):
+ """
+ Convert a list of integers to an array of integers.
+ """
+ return np.array(arr)
+
+
+def test_find_array():
+ from vllm.model_executor.models.gritlm import GritLMMeanPool
+
+ model_config = ModelConfig(
+ MODEL_NAME,
+ runner="pooling",
+ dtype="bfloat16",
+ seed=0,
+ )
+ pooling = GritLMMeanPool(model_config=model_config)
+
+ arr = _arr([0, 1, 2, 3, 4, 5, 6, 7, 8, 9])
+
+ assert pooling._find_array(arr, _arr([3, 4, 5]), start_idx=0) == 3
+ assert pooling._find_array(arr, _arr([3, 4, 5]), start_idx=1) == 3
+ assert pooling._find_array(arr, _arr([3, 4, 5]), start_idx=5) == -1
+ assert pooling._find_array(arr, _arr([3, 4, 5]), end_idx=3) == -1
+ assert pooling._find_array(arr, _arr([3, 4, 5]), end_idx=4) == 3
+ assert pooling._find_array(arr, _arr([3, 5]), start_idx=0) == -1
+
+ with pytest.raises(ValueError):
+ pooling._find_array(arr, _arr([3, 4, 5]), start_idx=-1)
+
+
+def run_llm_encode(
+ llm: LLM,
+ queries: list[str],
+ instruction: str,
+) -> list[list[float]]:
+ outputs = llm.embed([instruction + q for q in queries])
+ return [output.outputs.embedding for output in outputs]
+
+
+def gritlm_instruction(instruction):
+ return (
+ "<|user|>\n" + instruction + "\n<|embed|>\n" if instruction else "<|embed|>\n"
+ )
+
+
+def get_test_data():
+ """
+ Grabbed this test data and the expected values from
+ README.md in https://github.com/ContextualAI/gritlm
+ """
+ q_instruction = gritlm_instruction(
+ "Given a scientific paper title, retrieve the paper's abstract",
+ )
+ queries = [
+ "Bitcoin: A Peer-to-Peer Electronic Cash System",
+ "Generative Representational Instruction Tuning",
+ ]
+
+ d_instruction = gritlm_instruction("")
+ documents = [
+ # ruff: noqa: E501
+ "A purely peer-to-peer version of electronic cash would allow online payments to be sent directly from one party to another without going through a financial institution. Digital signatures provide part of the solution, but the main benefits are lost if a trusted third party is still required to prevent double-spending. We propose a solution to the double-spending problem using a peer-to-peer network. The network timestamps transactions by hashing them into an ongoing chain of hash-based proof-of-work, forming a record that cannot be changed without redoing the proof-of-work. The longest chain not only serves as proof of the sequence of events witnessed, but proof that it came from the largest pool of CPU power. As long as a majority of CPU power is controlled by nodes that are not cooperating to attack the network, they'll generate the longest chain and outpace attackers. The network itself requires minimal structure. Messages are broadcast on a best effort basis, and nodes can leave and rejoin the network at will, accepting the longest proof-of-work chain as proof of what happened while they were gone.",
+ "All text-based language problems can be reduced to either generation or embedding. Current models only perform well at one or the other. We introduce generative representational instruction tuning (GRIT) whereby a large language model is trained to handle both generative and embedding tasks by distinguishing between them through instructions. Compared to other open models, our resulting GritLM 7B sets a new state of the art on the Massive Text Embedding Benchmark (MTEB) and outperforms all models up to its size on a range of generative tasks. By scaling up further, GritLM 8X7B outperforms all open generative language models that we tried while still being among the best embedding models. Notably, we find that GRIT matches training on only generative or embedding data, thus we can unify both at no performance loss. Among other benefits, the unification via GRIT speeds up Retrieval-Augmented Generation (RAG) by > 60% for long documents, by no longer requiring separate retrieval and generation models. Models, code, etc. are freely available at https://github.com/ContextualAI/gritlm.",
+ ]
+
+ return queries, q_instruction, documents, d_instruction
+
+
+def validate_embed_output(q_rep: list[list[float]], d_rep: list[list[float]]):
+ cosine_sim_q0_d0 = 1 - cosine(q_rep[0], d_rep[0])
+ assert cosine_sim_q0_d0 == pytest.approx(0.609, abs=ATOL)
+
+ cosine_sim_q0_d1 = 1 - cosine(q_rep[0], d_rep[1])
+ assert cosine_sim_q0_d1 == pytest.approx(0.101, abs=ATOL)
+
+ cosine_sim_q1_d0 = 1 - cosine(q_rep[1], d_rep[0])
+ assert cosine_sim_q1_d0 == pytest.approx(0.120, abs=ATOL)
+
+ cosine_sim_q1_d1 = 1 - cosine(q_rep[1], d_rep[1])
+ assert cosine_sim_q1_d1 == pytest.approx(0.534, abs=ATOL)
+
+
+def test_gritlm_offline_embedding(vllm_runner):
+ queries, q_instruction, documents, d_instruction = get_test_data()
+
+ with vllm_runner(
+ MODEL_NAME,
+ runner="pooling",
+ max_model_len=MAX_MODEL_LEN,
+ ) as vllm_model:
+ llm = vllm_model.llm
+
+ d_rep = run_llm_encode(
+ llm,
+ documents,
+ d_instruction,
+ )
+ q_rep = run_llm_encode(
+ llm,
+ queries,
+ q_instruction,
+ )
+
+ validate_embed_output(q_rep, d_rep)
+
+
+@pytest.mark.asyncio
+async def test_gritlm_api_server_embedding():
+ queries, q_instruction, documents, d_instruction = get_test_data()
+
+ args = [
+ "--runner",
+ "pooling",
+ "--max_model_len",
+ str(MAX_MODEL_LEN),
+ *ROCM_EXTRA_ARGS,
+ ]
+
+ with RemoteOpenAIServer(MODEL_NAME, args, env_dict=ROCM_ENV_OVERRIDES) as server:
+ client_embedding = server.get_async_client()
+
+ d_rep = await run_client_embeddings(
+ client_embedding,
+ MODEL_NAME,
+ documents,
+ d_instruction,
+ )
+ q_rep = await run_client_embeddings(
+ client_embedding,
+ MODEL_NAME,
+ queries,
+ q_instruction,
+ )
+
+ validate_embed_output(q_rep, d_rep)
+
+
+def test_gritlm_offline_generate(monkeypatch: pytest.MonkeyPatch, vllm_runner):
+ input = "<|user|>\nWhat is the capital of France?\n<|assistant|>\n"
+
+ with vllm_runner(
+ MODEL_NAME,
+ runner="generate",
+ max_model_len=MAX_MODEL_LEN,
+ ) as vllm_model:
+ llm = vllm_model.llm
+
+ sampling_params = SamplingParams(temperature=0.0, max_tokens=256)
+ outputs = llm.generate(input, sampling_params=sampling_params)
+
+ assert outputs[0].outputs[0].text == "The capital of France is Paris."
+
+
+@pytest.mark.asyncio
+async def test_gritlm_api_server_generate():
+ input = "<|user|>\nWhat is the capital of France?\n<|assistant|>\n"
+
+ args = ["--runner", "generate", "--max_model_len", str(MAX_MODEL_LEN)]
+
+ with RemoteOpenAIServer(MODEL_NAME, args) as server:
+ client_generate = server.get_async_client()
+
+ outputs = await client_generate.completions.create(
+ model=MODEL_NAME,
+ prompt=input,
+ max_tokens=256,
+ temperature=0.0,
+ )
+
+ assert outputs.choices[0].text == "The capital of France is Paris."
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_head_dtype.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_head_dtype.py
new file mode 100644
index 0000000000000000000000000000000000000000..841445513de98d9884ddc5bf0c82dc5f31c41dcb
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_head_dtype.py
@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+import torch
+from transformers import AutoModelForSequenceClassification
+
+
+@pytest.mark.parametrize(
+ "model",
+ ["nie3e/sentiment-polish-gpt2-small"],
+)
+@pytest.mark.parametrize("dtype", ["half"])
+def test_classify_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ with hf_runner(
+ model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
+ ) as hf_model:
+ hf_outputs = hf_model.classify(example_prompts)
+
+ for head_dtype_str in ["float32", "model"]:
+ with vllm_runner(
+ model,
+ max_model_len=512,
+ dtype=dtype,
+ hf_overrides={"head_dtype": head_dtype_str},
+ ) as vllm_model:
+ model_config = vllm_model.llm.llm_engine.model_config
+ model_dtype = model_config.dtype
+ head_dtype = model_config.head_dtype
+
+ if head_dtype_str == "float32":
+ assert head_dtype == torch.float32
+ elif head_dtype_str == "model":
+ assert head_dtype == model_dtype
+
+ vllm_outputs = vllm_model.classify(example_prompts)
+
+ for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+ hf_output = torch.tensor(hf_output).float()
+ vllm_output = torch.tensor(vllm_output).float()
+
+ assert torch.allclose(hf_output, vllm_output, atol=1e-2)
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_jina_reranker_v3.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_jina_reranker_v3.py
new file mode 100644
index 0000000000000000000000000000000000000000..78e7ce3abf42db1e5128565f7cfae3a745152c20
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_jina_reranker_v3.py
@@ -0,0 +1,275 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# ruff: noqa: E501
+import pytest
+import requests
+import torch
+import torch.nn.functional as F
+
+from tests.utils import RemoteOpenAIServer
+from vllm.entrypoints.pooling.pooling.protocol import PoolingResponse
+from vllm.entrypoints.pooling.scoring.protocol import ScoreResponse
+
+model_name = "jinaai/jina-reranker-v3"
+query = "What are the health benefits of green tea?"
+documents = [
+ "Green tea contains antioxidants called catechins that may help reduce inflammation and protect cells from damage.",
+ "El precio del café ha aumentado un 20% este año debido a problemas en la cadena de suministro.",
+ "Studies show that drinking green tea regularly can improve brain function and boost metabolism.",
+ "Basketball is one of the most popular sports in the United States.",
+ "绿茶富含儿茶素等抗氧化剂,可以降低心脏病风险,还有助于控制体重。",
+ "Le thé vert est riche en antioxydants et peut améliorer la fonction cérébrale.",
+]
+
+EMBEDDING_SIZE = 512
+REFERENCE_1_VS_1 = [
+ 0.345703125,
+ -0.10498046,
+ 0.314453125,
+ -0.1376953125,
+ 0.3398437500,
+ 0.2539062,
+]
+REFERENCE_1_VS_N = [
+ 0.294921875,
+ -0.16015625,
+ 0.189453125,
+ -0.1708984375,
+ 0.2255859375,
+ 0.1640625,
+]
+TOL = 0.01
+
+
+def test_offline(vllm_runner):
+ with vllm_runner(model_name, runner="pooling") as llm_runner:
+ llm = llm_runner.get_llm()
+ _test_offline_1_v_1(llm)
+ _test_offline_1_v_n(llm)
+ _test_offline_n_v_n(llm)
+ _test_offline_token_embed_illegal_inputs(llm)
+ assert llm.model_config.embedding_size == EMBEDDING_SIZE
+
+
+def test_online():
+ with RemoteOpenAIServer(model_name, ["--runner", "pooling"]) as server:
+ _test_online_1_v_1(server)
+ _test_online_1_v_n(server)
+ _test_online_n_v_n(server)
+ _test_online_token_embed_illegal_inputs(server)
+
+
+def _test_offline_1_v_1(llm):
+ # test llm.score
+ outputs = llm.score(query, documents[0])
+ assert len(outputs) == 1
+ assert outputs[0].outputs.score == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
+
+ # test llm.encode
+ outputs = llm.encode(documents[:1] + [query], pooling_task="token_embed")
+ embeds = outputs[0].outputs.data.float()
+ assert embeds.shape[0] == 2
+ assert embeds.shape[-1] == EMBEDDING_SIZE
+
+ doc_embeds = embeds[:-1]
+ query_embeds = embeds[-1]
+
+ scores = F.cosine_similarity(query_embeds, doc_embeds)
+ assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
+
+
+def _test_offline_1_v_n(llm):
+ # test llm.score
+ outputs = llm.score(query, documents)
+ assert len(outputs) == len(documents)
+
+ for expected, output in zip(REFERENCE_1_VS_N, outputs):
+ actual = output.outputs.score
+ assert actual == pytest.approx(expected, abs=TOL)
+
+ # test llm.encode
+ outputs = llm.encode(documents + [query], pooling_task="token_embed")
+ embeds = outputs[0].outputs.data.float()
+ assert embeds.shape[0] == len(documents) + 1
+
+ doc_embeds = embeds[:-1]
+ query_embeds = embeds[-1]
+
+ scores = F.cosine_similarity(query_embeds, doc_embeds)
+
+ assert len(scores) == len(documents)
+ for expected, actual in zip(REFERENCE_1_VS_N, scores):
+ assert actual == pytest.approx(expected, abs=TOL)
+
+
+def _test_offline_n_v_n(llm):
+ # test llm.score
+ outputs = llm.score([query] * len(documents), documents)
+ assert len(outputs) == len(documents)
+
+ for expected, output in zip(REFERENCE_1_VS_1, outputs):
+ actual = output.outputs.score
+ assert actual == pytest.approx(expected, abs=TOL)
+
+ # test llm.encode
+ for doc, expected in zip(documents, REFERENCE_1_VS_1):
+ outputs = llm.encode([doc, query], pooling_task="token_embed")
+ embeds = outputs[0].outputs.data.float()
+ assert embeds.shape[0] == 2
+
+ doc_embeds = embeds[:-1]
+ query_embeds = embeds[-1]
+
+ scores = F.cosine_similarity(query_embeds, doc_embeds)
+ assert scores[0] == pytest.approx(expected, abs=TOL)
+
+
+def _test_offline_token_embed_illegal_inputs(llm):
+ with pytest.raises(
+ ValueError, match="The JinaForRanking model requires at least 2 inputs."
+ ):
+ llm.encode([query], pooling_task="token_embed")
+
+ with pytest.raises(
+ ValueError, match="The JinaForRanking model only supports text as input."
+ ):
+ llm.encode([1, 2, 3], pooling_task="token_embed")
+
+
+def _get_scores(server, query, document):
+ score_response = requests.post(
+ server.url_for("score"),
+ json={
+ "model": model_name,
+ "queries": query,
+ "documents": document,
+ },
+ )
+
+ score_response.raise_for_status()
+ score = ScoreResponse.model_validate(score_response.json())
+
+ return [d.score for d in score.data]
+
+
+def _get_embeds(server, prompts: list[str]):
+ response = requests.post(
+ server.url_for("pooling"),
+ json={
+ "model": model_name,
+ "task": "token_embed",
+ "input": prompts,
+ "encoding_format": "float",
+ },
+ )
+ response.raise_for_status()
+ poolings = PoolingResponse.model_validate(response.json())
+
+ return torch.as_tensor([d.data for d in poolings.data][0]).float()
+
+
+def _test_online_1_v_1(server):
+ # test scoring api
+ scores = _get_scores(server, query, documents[0])
+ assert len(scores) == 1
+ assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
+
+ # test pooling api
+ embeds = _get_embeds(server, [documents[0], query])
+ assert embeds.shape[0] == 2
+ assert embeds.shape[-1] == EMBEDDING_SIZE
+
+ doc_embeds = embeds[:-1]
+ query_embeds = embeds[-1]
+
+ scores = F.cosine_similarity(query_embeds, doc_embeds)
+ assert scores[0] == pytest.approx(REFERENCE_1_VS_1[0], abs=TOL)
+
+
+def _test_online_1_v_n(server):
+ # test scoring api
+ scores = _get_scores(server, query, documents)
+ assert len(scores) == len(documents)
+
+ for expected, actual in zip(REFERENCE_1_VS_N, scores):
+ assert actual == pytest.approx(expected, abs=TOL)
+
+ # test pooling api
+ embeds = _get_embeds(server, documents + [query])
+ assert embeds.shape[0] == len(documents) + 1
+
+ doc_embeds = embeds[:-1]
+ query_embeds = embeds[-1]
+
+ scores = F.cosine_similarity(query_embeds, doc_embeds)
+
+ assert len(scores) == len(documents)
+ for expected, actual in zip(REFERENCE_1_VS_N, scores):
+ assert actual == pytest.approx(expected, abs=TOL)
+
+
+def _test_online_n_v_n(server):
+ # test scoring api
+ scores = _get_scores(server, [query] * len(documents), documents)
+ assert len(scores) == len(documents)
+
+ for expected, actual in zip(REFERENCE_1_VS_1, scores):
+ assert actual == pytest.approx(expected, abs=TOL)
+
+ # test pooling api
+ for doc, expected in zip(documents, REFERENCE_1_VS_1):
+ embeds = _get_embeds(server, [doc, query])
+ assert embeds.shape[0] == 2
+
+ doc_embeds = embeds[:-1]
+ query_embeds = embeds[-1]
+
+ scores = F.cosine_similarity(query_embeds, doc_embeds)
+ assert len(scores) == 1
+ assert scores[0] == pytest.approx(expected, abs=TOL)
+
+
+def _test_online_token_embed_illegal_inputs(server):
+ response = requests.post(
+ server.url_for("pooling"),
+ json={
+ "model": model_name,
+ "task": "token_embed",
+ "input": [query],
+ "encoding_format": "float",
+ },
+ )
+ assert response.json()["error"]["message"].startswith(
+ "The JinaForRanking model requires at least 2 inputs."
+ )
+
+ response = requests.post(
+ server.url_for("pooling"),
+ json={
+ "model": model_name,
+ "task": "token_embed",
+ "input": [1, 2, 3],
+ "encoding_format": "float",
+ },
+ )
+ assert response.json()["error"]["message"].startswith(
+ "The JinaForRanking model only supports text as input."
+ )
+
+ response = requests.post(
+ server.url_for("pooling"),
+ json={
+ "model": model_name,
+ "task": "token_embed",
+ "messages": [
+ {
+ "role": "user",
+ "content": "The cat sat on the mat.",
+ }
+ ],
+ "encoding_format": "float",
+ },
+ )
+ assert response.json()["error"]["message"].startswith(
+ "The JinaForRanking does not support chat Request."
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_max_tokens_per_doc.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_max_tokens_per_doc.py
new file mode 100644
index 0000000000000000000000000000000000000000..7596cbb162c8570937a8ac845f813d1df8df37b2
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_max_tokens_per_doc.py
@@ -0,0 +1,228 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""
+Tests for max_tokens_per_doc and max_tokens_per_query.
+"""
+
+import json
+import os
+from dataclasses import dataclass
+
+import pytest
+import requests
+
+from tests.utils import VLLM_PATH, RemoteOpenAIServer
+from vllm.entrypoints.pooling.scoring.protocol import RerankResponse
+
+os.environ["VLLM_LOGGING_LEVEL"] = "WARNING"
+
+TEMPLATE_DIR = str(VLLM_PATH / "examples/pooling/score/template")
+ExpectedPromptTokens = int | tuple[int, ...]
+
+long_query = "What is the capital of France?" * 20
+long_doc = "The capital of France is Paris. " * 20
+
+
+@dataclass
+class TestConfig:
+ model: str
+ args: list[str]
+ without_truncated_prompt_tokens: ExpectedPromptTokens
+ with_max_tokens_per_query_prompt_tokens: ExpectedPromptTokens
+ with_max_tokens_per_doc_prompt_tokens: ExpectedPromptTokens
+ with_max_tokens_per_query_and_doc_prompt_tokens: ExpectedPromptTokens
+
+
+RERANK_CONFIGS = [
+ # 1. cross-encoder
+ TestConfig(
+ model="jinaai/jina-reranker-v2-base-multilingual",
+ args=[
+ "--enforce-eager",
+ "--max-model-len",
+ "1024",
+ "--trust-remote-code",
+ ],
+ without_truncated_prompt_tokens=284,
+ with_max_tokens_per_query_prompt_tokens=154,
+ with_max_tokens_per_doc_prompt_tokens=154,
+ with_max_tokens_per_query_and_doc_prompt_tokens=24,
+ ),
+ # 2. cross-encoder + score template
+ TestConfig(
+ model="Qwen/Qwen3-Reranker-0.6B",
+ args=[
+ "--enforce-eager",
+ "--max-model-len",
+ "1024",
+ "--hf-overrides",
+ json.dumps(
+ {
+ "architectures": ["Qwen3ForSequenceClassification"],
+ "classifier_from_token": ["no", "yes"],
+ "is_original_qwen3_reranker": True,
+ }
+ ),
+ "--chat-template",
+ os.path.join(TEMPLATE_DIR, "qwen3_reranker.jinja"),
+ ],
+ without_truncated_prompt_tokens=352,
+ with_max_tokens_per_query_prompt_tokens=223,
+ with_max_tokens_per_doc_prompt_tokens=221,
+ with_max_tokens_per_query_and_doc_prompt_tokens=92,
+ ),
+ # 3. bi-encoder
+ TestConfig(
+ model="intfloat/multilingual-e5-small",
+ args=[
+ "--enforce-eager",
+ "--max-model-len",
+ "512",
+ "--trust-remote-code",
+ ],
+ # This model has produced both prompt-token totals in CI/local cache;
+ # keep truncation checks exact while tolerating the boundary delta.
+ without_truncated_prompt_tokens=(285, 286),
+ with_max_tokens_per_query_prompt_tokens=(155, 156),
+ with_max_tokens_per_doc_prompt_tokens=155,
+ with_max_tokens_per_query_and_doc_prompt_tokens=25,
+ ),
+ # 4. late-interaction
+ TestConfig(
+ model="answerdotai/answerai-colbert-small-v1",
+ args=[
+ "--enforce-eager",
+ "--max-model-len",
+ "512",
+ "--trust-remote-code",
+ ],
+ without_truncated_prompt_tokens=285,
+ with_max_tokens_per_query_prompt_tokens=155,
+ with_max_tokens_per_doc_prompt_tokens=155,
+ with_max_tokens_per_query_and_doc_prompt_tokens=25,
+ ),
+ # 5. jinaai/jina-reranker-v3
+ TestConfig(
+ model="jinaai/jina-reranker-v3",
+ args=[
+ "--enforce-eager",
+ "--max-model-len",
+ "1024",
+ "--trust-remote-code",
+ ],
+ without_truncated_prompt_tokens=567,
+ with_max_tokens_per_query_prompt_tokens=308,
+ with_max_tokens_per_doc_prompt_tokens=436,
+ with_max_tokens_per_query_and_doc_prompt_tokens=177,
+ ),
+]
+
+
+def assert_prompt_tokens(actual: int, expected: ExpectedPromptTokens) -> None:
+ if isinstance(expected, int):
+ assert actual == expected
+ else:
+ assert actual in expected
+
+
+@pytest.fixture(scope="module", params=RERANK_CONFIGS, ids=lambda c: c.model)
+def server(request):
+ config: TestConfig = request.param
+ with RemoteOpenAIServer(config.model, config.args) as remote_server:
+ yield config, remote_server
+
+
+def test_without_truncated(server):
+ """Test that max_tokens_per_doc truncates documents correctly."""
+ config, remote_server = server
+
+ response = requests.post(
+ remote_server.url_for("rerank"),
+ json={"model": config.model, "query": long_query, "documents": [long_doc]},
+ )
+ response.raise_for_status()
+ rerank = RerankResponse.model_validate(response.json())
+
+ assert rerank.id is not None
+ assert rerank.results is not None
+ assert len(rerank.results) == 1
+ assert_prompt_tokens(
+ rerank.usage.prompt_tokens,
+ config.without_truncated_prompt_tokens,
+ )
+
+
+def test_max_tokens_per_query(server):
+ """Test that max_tokens_per_doc truncates documents correctly."""
+ config, remote_server = server
+
+ response = requests.post(
+ remote_server.url_for("rerank"),
+ json={
+ "model": config.model,
+ "query": long_query,
+ "documents": [long_doc],
+ "max_tokens_per_query": 10,
+ },
+ )
+ response.raise_for_status()
+ rerank = RerankResponse.model_validate(response.json())
+
+ assert rerank.id is not None
+ assert rerank.results is not None
+ assert len(rerank.results) == 1
+ assert_prompt_tokens(
+ rerank.usage.prompt_tokens,
+ config.with_max_tokens_per_query_prompt_tokens,
+ )
+
+
+def test_max_tokens_per_doc(server):
+ """Test that max_tokens_per_doc truncates documents correctly."""
+ config, remote_server = server
+
+ response = requests.post(
+ remote_server.url_for("rerank"),
+ json={
+ "model": config.model,
+ "query": long_query,
+ "documents": [long_doc],
+ "max_tokens_per_doc": 10,
+ },
+ )
+ response.raise_for_status()
+ rerank = RerankResponse.model_validate(response.json())
+
+ assert rerank.id is not None
+ assert rerank.results is not None
+ assert len(rerank.results) == 1
+ assert_prompt_tokens(
+ rerank.usage.prompt_tokens,
+ config.with_max_tokens_per_doc_prompt_tokens,
+ )
+
+
+def test_max_tokens_per_query_and_doc(server):
+ """Test that max_tokens_per_doc truncates documents correctly."""
+ config, remote_server = server
+
+ response = requests.post(
+ remote_server.url_for("rerank"),
+ json={
+ "model": config.model,
+ "query": long_query,
+ "documents": [long_doc],
+ "max_tokens_per_query": 10,
+ "max_tokens_per_doc": 10,
+ },
+ )
+ response.raise_for_status()
+ rerank = RerankResponse.model_validate(response.json())
+
+ assert rerank.id is not None
+ assert rerank.results is not None
+ assert len(rerank.results) == 1
+ assert_prompt_tokens(
+ rerank.usage.prompt_tokens,
+ config.with_max_tokens_per_query_and_doc_prompt_tokens,
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_mm_classifier_conversion.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_mm_classifier_conversion.py
new file mode 100644
index 0000000000000000000000000000000000000000..b0e5f841b1f58ca864f8b527c46f0b73d7f95c1f
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_mm_classifier_conversion.py
@@ -0,0 +1,104 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from vllm.config.pooler import PoolerConfig
+
+
+def test_idefics_multimodal(
+ vllm_runner,
+) -> None:
+ prompts = [
+ "Hello, my name is",
+ "The president of the United States is",
+ "The capital of France is",
+ "The future of AI is",
+ ]
+
+ with vllm_runner(
+ model_name="HuggingFaceM4/Idefics3-8B-Llama3",
+ runner="pooling",
+ convert="classify",
+ load_format="dummy",
+ max_model_len=512,
+ enforce_eager=True,
+ tensor_parallel_size=1,
+ disable_log_stats=True,
+ dtype="bfloat16",
+ ) as vllm_model:
+ llm = vllm_model.get_llm()
+ outputs = llm.classify(prompts)
+ for output in outputs:
+ assert len(output.outputs.probs) == 2
+
+
+def update_config(config):
+ text_config = config.get_text_config()
+ text_config.update(
+ {
+ "architectures": ["Gemma3ForSequenceClassification"],
+ "classifier_from_token": ["A", "B", "C", "D", "E"],
+ "method": "no_post_processing",
+ "id2label": {
+ "A": "Chair",
+ "B": "Couch",
+ "C": "Table",
+ "D": "Bed",
+ "E": "Cupboard",
+ },
+ }
+ )
+ return config
+
+
+def test_gemma_multimodal(
+ vllm_runner,
+) -> None:
+ messages = [
+ {
+ "role": "system",
+ "content": """
+ You are a helpful assistant. You will be given a product description
+ which may also include an image. Classify the following product into
+ one of the categories:
+
+ A = chair
+ B = couch
+ C = table
+ D = bed
+ E = cupboard
+
+ You'll answer with exactly one letter (A, B, C, D, or E).""",
+ },
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "image_url",
+ "image_url": {
+ "url": "https://vllm-public-assets.s3.us-west-2.amazonaws.com/multimodal_asset/red_chair.jpg"
+ },
+ },
+ {"type": "text", "text": "A fine 19th century piece of furniture."},
+ ],
+ },
+ ]
+
+ with vllm_runner(
+ model_name="google/gemma-3-4b-it",
+ runner="pooling",
+ convert="classify",
+ load_format="auto",
+ hf_overrides=update_config,
+ pooler_config=PoolerConfig(seq_pooling_type="LAST"),
+ max_model_len=512,
+ enforce_eager=True,
+ tensor_parallel_size=1,
+ disable_log_stats=True,
+ dtype="bfloat16",
+ ) as vllm_model:
+ llm = vllm_model.get_llm()
+ prompts = llm._preprocess_chat([messages])
+
+ result = llm.classify(prompts)
+ assert result[0].outputs.probs[0] > 0.95
+ assert all(c < 0.05 for c in result[0].outputs.probs[1:])
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_multi_vector_retrieval.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_multi_vector_retrieval.py
new file mode 100644
index 0000000000000000000000000000000000000000..9d571ba146599fedeec3fb79395cd74bea972af0
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_multi_vector_retrieval.py
@@ -0,0 +1,47 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+import torch
+from transformers import AutoModel
+
+from tests.models.utils import check_embeddings_close
+from vllm.config import PoolerConfig
+
+
+@pytest.mark.parametrize(
+ "model",
+ ["BAAI/bge-m3"],
+)
+@pytest.mark.parametrize("dtype", ["half"])
+@torch.inference_mode
+def test_embed_models(hf_runner, vllm_runner, example_prompts, model: str, dtype: str):
+ with vllm_runner(
+ model,
+ runner="pooling",
+ pooler_config=PoolerConfig(task="token_embed"),
+ max_model_len=None,
+ ) as vllm_model:
+ vllm_outputs = vllm_model.token_embed(example_prompts)
+
+ with hf_runner(
+ model,
+ auto_cls=AutoModel,
+ ) as hf_model:
+ tokenizer = hf_model.tokenizer
+ hf_outputs = []
+ for prompt in example_prompts:
+ inputs = tokenizer([prompt], return_tensors="pt")
+ inputs = hf_model.wrap_device(inputs)
+ output = hf_model.model(**inputs)
+ embedding = output.last_hidden_state[0].float()
+ # normal
+ hf_outputs.append(embedding.cpu())
+
+ for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+ check_embeddings_close(
+ embeddings_0_lst=hf_output,
+ embeddings_1_lst=vllm_output,
+ name_0="hf",
+ name_1="vllm",
+ tol=1e-2,
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_multilabel_classification_support.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_multilabel_classification_support.py
new file mode 100644
index 0000000000000000000000000000000000000000..c45581bcf96db8c77330f44ec5936db39ec38877
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_multilabel_classification_support.py
@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+import torch
+from transformers import AutoModelForSequenceClassification
+
+
+@pytest.mark.parametrize(
+ "model",
+ ["Rami/multi-label-class-classification-on-github-issues"],
+)
+@pytest.mark.parametrize("dtype", ["half"])
+def test_classify_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ with vllm_runner(model, max_model_len=512, dtype=dtype) as vllm_model:
+ vllm_outputs = vllm_model.classify(example_prompts)
+
+ with hf_runner(
+ model, dtype=dtype, auto_cls=AutoModelForSequenceClassification
+ ) as hf_model:
+ hf_outputs = hf_model.classify(example_prompts)
+
+ for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+ hf_output = torch.tensor(hf_output)
+ vllm_output = torch.tensor(vllm_output)
+
+ assert torch.allclose(
+ hf_output, vllm_output, 1e-3 if dtype == "float" else 1e-2
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_nomic_max_model_len.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_nomic_max_model_len.py
new file mode 100644
index 0000000000000000000000000000000000000000..e900fac13cf0e9c7bf24de745a69f15d4880f9b0
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_nomic_max_model_len.py
@@ -0,0 +1,111 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# ruff: noqa: SIM117
+
+import pytest
+
+from ...utils import EmbedModelInfo
+
+MODELS = [
+ EmbedModelInfo(
+ "nomic-ai/nomic-embed-text-v1",
+ # Fixme:
+ # Update nomic-embed code to support the latest
+ # HF version and remove revision set.
+ revision="720244025c1a7e15661a174c63cce63c8218e52b",
+ ),
+ # EmbedModelInfo("nomic-ai/nomic-embed-text-v1.5"),
+ # EmbedModelInfo("nomic-ai/CodeRankEmbed"),
+ EmbedModelInfo("nomic-ai/nomic-embed-text-v2-moe"),
+ # EmbedModelInfo("Snowflake/snowflake-arctic-embed-m-long"),
+]
+
+rope_theta = 1000
+factor = 4.0
+original_max_position_embeddings = 2048
+max_model_len = int(original_max_position_embeddings * factor)
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_default(model_info, vllm_runner):
+ with vllm_runner(
+ model_info.name,
+ revision=model_info.revision,
+ runner="pooling",
+ max_model_len=None,
+ ) as vllm_model:
+ model_config = vllm_model.llm.llm_engine.model_config
+ if model_info.name == "nomic-ai/nomic-embed-text-v2-moe":
+ # For nomic-embed-text-v2-moe the length is set to 512
+ # by sentence_bert_config.json.
+ assert model_config.max_model_len == 512
+ if model_info.name == "nomic-ai/nomic-embed-text-v1":
+ assert model_config.max_model_len == 8192
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_set_max_model_len_legal(model_info, vllm_runner):
+ # set max_model_len <= 512
+ with vllm_runner(
+ model_info.name,
+ revision=model_info.revision,
+ runner="pooling",
+ max_model_len=256,
+ ) as vllm_model:
+ model_config = vllm_model.llm.llm_engine.model_config
+ assert model_config.max_model_len == 256
+
+ # For nomic-embed-text-v2-moe the length is set to 512
+ # by sentence_bert_config.json.
+ if model_info.name == "nomic-ai/nomic-embed-text-v2-moe":
+ with pytest.raises(ValueError):
+ with vllm_runner(
+ model_info.name,
+ revision=model_info.revision,
+ runner="pooling",
+ max_model_len=1024,
+ ):
+ pass
+ return
+
+ # set 512 < max_model_len <= 2048
+ with vllm_runner(
+ model_info.name,
+ revision=model_info.revision,
+ runner="pooling",
+ max_model_len=1024,
+ ) as vllm_model:
+ model_config = vllm_model.llm.llm_engine.model_config
+ assert model_config.max_model_len == 1024
+
+ # set max_model_len > 2048
+ with vllm_runner(
+ model_info.name,
+ revision=model_info.revision,
+ runner="pooling",
+ max_model_len=4096,
+ ) as vllm_model:
+ model_config = vllm_model.llm.llm_engine.model_config
+ assert model_config.max_model_len == 4096
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_use_rope_scaling_legal(model_info, vllm_runner):
+ hf_overrides = {
+ "rope_parameters": {
+ "rope_theta": rope_theta,
+ "rope_type": "yarn",
+ "factor": factor,
+ "original_max_position_embeddings": original_max_position_embeddings,
+ },
+ "max_model_len": max_model_len,
+ }
+
+ with vllm_runner(
+ model_info.name,
+ revision=model_info.revision,
+ runner="pooling",
+ max_model_len=None,
+ hf_overrides=hf_overrides,
+ ):
+ pass
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_pooler_config_init_behaviour.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_pooler_config_init_behaviour.py
new file mode 100644
index 0000000000000000000000000000000000000000..865145a900390ea207e50a143bdc1b7999eff045
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_pooler_config_init_behaviour.py
@@ -0,0 +1,167 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+import torch
+import torch.nn.functional as F
+
+from tests.models.utils import softmax
+from vllm.config import PoolerConfig
+
+
+@pytest.mark.parametrize(
+ "model",
+ ["jason9693/Qwen2.5-1.5B-apeach", "papluca/xlm-roberta-base-language-detection"],
+)
+@pytest.mark.parametrize("dtype", ["half"])
+def test_classify_models_using_activation(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ with vllm_runner(
+ model,
+ max_model_len=512,
+ dtype=dtype,
+ pooler_config=PoolerConfig(use_activation=False),
+ ) as vllm_model:
+ wo_activation_out = vllm_model.classify(example_prompts)
+
+ with vllm_runner(
+ model,
+ max_model_len=512,
+ dtype=dtype,
+ pooler_config=PoolerConfig(use_activation=True),
+ ) as vllm_model:
+ w_activation_out = vllm_model.classify(example_prompts)
+
+ for wo_activation, w_activation in zip(wo_activation_out, w_activation_out):
+ wo_activation = torch.tensor(wo_activation)
+ w_activation = torch.tensor(w_activation)
+
+ assert not torch.allclose(wo_activation, w_activation, atol=1e-2), (
+ "pooler_config is not working"
+ )
+ assert torch.allclose(
+ softmax(wo_activation), w_activation, 1e-3 if dtype == "float" else 1e-2
+ )
+
+
+@pytest.mark.parametrize(
+ "model",
+ [
+ "intfloat/multilingual-e5-small",
+ ],
+)
+@pytest.mark.parametrize("dtype", ["half"])
+def test_embed_models_using_normalize(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ with vllm_runner(
+ model,
+ max_model_len=512,
+ dtype=dtype,
+ pooler_config=PoolerConfig(use_activation=False),
+ ) as vllm_model:
+ wo_normalize = torch.tensor(vllm_model.embed(example_prompts))
+
+ with vllm_runner(
+ model,
+ max_model_len=512,
+ dtype=dtype,
+ pooler_config=PoolerConfig(use_activation=True),
+ ) as vllm_model:
+ w_normalize = torch.tensor(vllm_model.embed(example_prompts))
+
+ assert not torch.allclose(wo_normalize, w_normalize, atol=1e-2), (
+ "pooler_config normalize is not working"
+ )
+ assert torch.allclose(
+ F.normalize(wo_normalize, p=2, dim=-1), w_normalize, atol=1e-2
+ ), "w_normal should be close to normal(wo_normal)."
+
+
+@pytest.mark.parametrize(
+ "model",
+ [
+ "internlm/internlm2-1_8b-reward",
+ ],
+)
+@pytest.mark.parametrize("dtype", ["half"])
+def test_reward_models_using_activation(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ with vllm_runner(
+ model,
+ max_model_len=1024,
+ dtype=dtype,
+ pooler_config=PoolerConfig(use_activation=False),
+ ) as vllm_model:
+ wo_activation = vllm_model.token_classify(example_prompts)
+
+ with vllm_runner(
+ model,
+ max_model_len=1024,
+ dtype=dtype,
+ pooler_config=PoolerConfig(use_activation=True),
+ ) as vllm_model:
+ w_activation = vllm_model.token_classify(example_prompts)
+
+ for wo, w in zip(wo_activation, w_activation):
+ wo = torch.tensor(wo)
+ w = torch.tensor(w)
+
+ assert not torch.allclose(wo, w, atol=1e-2), (
+ "pooler_config activation is not working"
+ )
+ assert torch.allclose(softmax(wo), w, atol=1e-2), (
+ "w_activation should be close to activation(wo_activation)."
+ )
+
+
+@pytest.mark.parametrize(
+ "model",
+ [
+ "intfloat/multilingual-e5-small",
+ ],
+)
+@pytest.mark.parametrize("dtype", ["half"])
+def test_multi_vector_retrieval_models_using_normalize(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ with vllm_runner(
+ model,
+ max_model_len=512,
+ dtype=dtype,
+ pooler_config=PoolerConfig(use_activation=False, task="token_embed"),
+ ) as vllm_model:
+ wo_normalize = vllm_model.token_embed(example_prompts)
+
+ with vllm_runner(
+ model,
+ max_model_len=512,
+ dtype=dtype,
+ pooler_config=PoolerConfig(use_activation=True, task="token_embed"),
+ ) as vllm_model:
+ w_normalize = vllm_model.token_embed(example_prompts)
+
+ for wo, w in zip(wo_normalize, w_normalize):
+ assert not torch.allclose(wo, w, atol=1e-2), (
+ "pooler_config normalize is not working"
+ )
+ assert torch.allclose(F.normalize(wo, p=2, dim=-1), w, atol=1e-2), (
+ "w_normal should be close to normal(wo_normal)."
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_reward.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_reward.py
new file mode 100644
index 0000000000000000000000000000000000000000..1d2da5e39afc303f7a9209a916c37d0b33928874
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_reward.py
@@ -0,0 +1,158 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import json
+from typing import TYPE_CHECKING
+
+import pytest
+import torch
+import torch.nn.functional as F
+from transformers import AutoModel
+
+from vllm.platforms import current_platform
+
+from ....conftest import HfRunner
+from ....utils import VLLM_PATH
+from ...registry import HF_EXAMPLE_MODELS
+
+if TYPE_CHECKING:
+ from _typeshed import StrPath
+
+
+FIXTURES_PATH = VLLM_PATH / "tests/models/fixtures"
+assert FIXTURES_PATH.exists()
+FIXTURE_REWARD_RESULT = {
+ "Qwen/Qwen2.5-Math-PRM-7B": FIXTURES_PATH / "qwen2_5_math_prm_reward_step.json",
+}
+
+
+@pytest.fixture
+def math_step_prompts():
+ # ruff: noqa: E501
+ data = {
+ "system": "Please reason step by step, and put your final answer within \\boxed{}. ",
+ "query": "Sue lives in a fun neighborhood. One weekend, the neighbors decided to play a prank on Sue. On Friday morning, the neighbors placed 18 pink plastic flamingos out on Sue's front yard. On Saturday morning, the neighbors took back one third of the flamingos, painted them white, and put these newly painted white flamingos back out on Sue's front yard. Then, on Sunday morning, they added another 18 pink plastic flamingos to the collection. At noon on Sunday, how many more pink plastic flamingos were out than white plastic flamingos?",
+ "response": [
+ "To find out how many more pink plastic flamingos were out than white plastic flamingos at noon on Sunday, we can break down the problem into steps. First, on Friday, the neighbors start with 18 pink plastic flamingos.",
+ "On Saturday, they take back one third of the flamingos. Since there were 18 flamingos, (1/3 \\times 18 = 6) flamingos are taken back. So, they have (18 - 6 = 12) flamingos left in their possession. Then, they paint these 6 flamingos white and put them back out on Sue's front yard. Now, Sue has the original 12 pink flamingos plus the 6 new white ones. Thus, by the end of Saturday, Sue has (12 + 6 = 18) pink flamingos and 6 white flamingos.",
+ "On Sunday, the neighbors add another 18 pink plastic flamingos to Sue's front yard. By the end of Sunday morning, Sue has (18 + 18 = 36) pink flamingos and still 6 white flamingos.",
+ "To find the difference, subtract the number of white flamingos from the number of pink flamingos: (36 - 6 = 30). Therefore, at noon on Sunday, there were 30 more pink plastic flamingos out than white plastic flamingos. The answer is (\\boxed{30}).",
+ ],
+ }
+ answer = "".join(data["response"]) + ""
+ prompt = f"system\n{data['system']}\nuser\n{data['query']}\nassistant\n{answer}<|endoftext|>"
+ return [prompt]
+
+
+def step_reward_patch_hf_model(hf_model: HfRunner):
+ # Patch the hf_runner to use the step reward function
+ def make_step_rewards(
+ logits: torch.Tensor, token_masks: torch.Tensor
+ ) -> list[list[float]]:
+ probabilities = F.softmax(logits, dim=-1)
+ probabilities = probabilities * token_masks.unsqueeze(-1)
+
+ all_scores_res: list[list[float]] = []
+ for i in range(probabilities.size(0)):
+ sample = probabilities[i] # seq_len, num_labels
+ positive_probs = sample[sample != 0].view(-1, 2)
+ non_zero_elements_list = positive_probs.cpu().tolist()
+ all_scores_res.append(non_zero_elements_list)
+ return all_scores_res
+
+ def reward(prompts: list[str]) -> list[list[float]]:
+ input_ids = hf_model.tokenizer(prompts, return_tensors="pt").input_ids
+ input_ids = hf_model.wrap_device(input_ids)
+ outputs = hf_model.model(input_ids=input_ids)
+
+ step_sep_id = hf_model.tokenizer.encode("")[0]
+ token_masks = input_ids == step_sep_id
+ return make_step_rewards(outputs[0], token_masks)
+
+ hf_model.reward = reward # type: ignore[attr-defined]
+
+ return hf_model
+
+
+def dump_reward_outputs(outputs: list[list[float]], filename: "StrPath"):
+ with open(filename, "w", encoding="utf-8") as f:
+ json.dump(outputs, f)
+
+
+def load_reward_outputs(filename: "StrPath") -> list[list[float]]:
+ with open(filename, encoding="utf-8") as f:
+ return json.load(f)
+
+
+@pytest.mark.parametrize(
+ "model",
+ [
+ pytest.param(
+ "Qwen/Qwen2.5-Math-PRM-7B",
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
+ ),
+ ],
+)
+@pytest.mark.parametrize("dtype", ["half"])
+def test_prm_models(
+ hf_runner,
+ vllm_runner,
+ math_step_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_transformers_version(on_fail="skip")
+
+ if current_platform.is_cpu():
+ pytest.skip("CPU only supports V1")
+
+ with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
+ vllm_outputs = vllm_model.token_classify(math_step_prompts)
+
+ with hf_runner(model, dtype=dtype, auto_cls=AutoModel) as hf_model:
+ hf_model = step_reward_patch_hf_model(hf_model)
+ hf_outputs = hf_model.reward(math_step_prompts)
+
+ dump_reward_outputs(
+ hf_outputs,
+ FIXTURE_REWARD_RESULT[model],
+ )
+
+ # check logits difference
+ for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+ hf_output = torch.tensor(hf_output).float()
+ vllm_output = torch.tensor(vllm_output).float()
+
+ assert torch.allclose(hf_output, vllm_output, 1.5e-2)
+
+
+@pytest.mark.parametrize(
+ "model",
+ [
+ pytest.param(
+ "Qwen/Qwen2.5-Math-PRM-7B",
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
+ ),
+ ],
+)
+@pytest.mark.parametrize("dtype", ["half"])
+def test_prm_models_with_golden_outputs(
+ vllm_runner,
+ math_step_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ if not FIXTURE_REWARD_RESULT.get(model):
+ pytest.skip(f"No available golden outputs for {model}.")
+
+ with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
+ vllm_outputs = vllm_model.token_classify(math_step_prompts)
+
+ golden_outputs = load_reward_outputs(FIXTURE_REWARD_RESULT[model])
+
+ # check logits difference
+ for golden_output, vllm_output in zip(golden_outputs, vllm_outputs):
+ golden_output = torch.tensor(golden_output).float()
+ vllm_output = torch.tensor(vllm_output).float()
+
+ assert torch.allclose(golden_output, vllm_output, 1.5e-2)
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_splade_sparse_pooler.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_splade_sparse_pooler.py
new file mode 100644
index 0000000000000000000000000000000000000000..4c40150511918c0ac019e49aeef680b8b6868dd4
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_splade_sparse_pooler.py
@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+import torch
+import torch.nn as nn
+
+from vllm.model_executor.models.bert import (
+ BertMLMHead,
+ SPLADESparsePooler,
+)
+from vllm.pooling_params import PoolingParams
+from vllm.v1.pool.metadata import PoolingMetadata, PoolingStates
+
+# ---------------------------------------------------------------------
+# Functional test: SPLADE formula correctness (no HF download needed)
+# ---------------------------------------------------------------------
+
+
+@pytest.mark.parametrize("B,T,H,V", [(2, 3, 5, 7)])
+@torch.inference_mode
+def test_splade_pooler_matches_reference_formula(B, T, H, V):
+ """Ensure SPLADESparsePooler forward() matches the mathematical formula:
+ log1p(relu(logits)) -> max over sequence length (after masking)."""
+ torch.manual_seed(0)
+
+ # Prepare [B] sequences of shape [T, H]
+ hs_list = [torch.randn(T, H) for _ in range(B)]
+ hs_tenser = torch.cat(hs_list)
+
+ # Simulate PoolingMetadata (only required fields)
+ prompt_lens = [T, T - 1]
+ prompt_lens_tenser = torch.tensor(prompt_lens, dtype=torch.int32)
+ token_ids = torch.tensor(
+ [
+ [101, 5, 102], # Batch 0: [CLS], token, [SEP]
+ [101, 6, 6], # Batch 1: [CLS], token, token (last token ignored)
+ ],
+ dtype=torch.long,
+ )
+ meta = PoolingMetadata(
+ prompt_lens=prompt_lens_tenser,
+ prompt_token_ids=token_ids,
+ prompt_token_ids_cpu=token_ids,
+ pooling_params=[PoolingParams(task="embed")] * B,
+ pooling_states=[PoolingStates() for _ in range(B)],
+ )
+
+ # MLM head (prefer BertMLMHead, fallback to Linear if unavailable)
+ try:
+ mlm_head = BertMLMHead(hidden_size=H, vocab_size=V, layer_norm_eps=1e-12)
+ except Exception:
+ mlm_head = nn.Linear(H, V, bias=True)
+
+ # Forward pass through SPLADE pooler
+ pooler = SPLADESparsePooler(mlm_head=mlm_head, pooling="max", remove_cls_sep=True)
+ pooled = pooler(hidden_states=hs_tenser, pooling_metadata=meta) # list of [V]
+
+ # Basic output checks
+ assert isinstance(pooled, torch.Tensor) and len(pooled) == B
+ for vec in pooled:
+ assert vec.shape == (V,)
+ assert torch.isfinite(vec).all()
+ assert (vec >= 0).all(), "SPLADE outputs must be non-negative."
+
+ # Reference implementation for comparison
+ def ref_one(hs: torch.Tensor, L: int, tid_row: torch.Tensor) -> torch.Tensor:
+ keep = torch.ones(L, dtype=torch.bool)
+ if L > 0 and tid_row[0].item() == 101: # remove CLS
+ keep[0] = False
+ if L > 0 and tid_row[L - 1].item() == 102: # remove SEP
+ keep[L - 1] = False
+
+ valid = hs[:L][keep[:L]]
+ if valid.numel() == 0:
+ return torch.zeros(V, dtype=torch.float32)
+
+ logits = mlm_head(valid) # [L', V]
+ scores = torch.log1p(torch.relu(logits)) # [L', V]
+ return scores.max(dim=0).values.to(torch.float32)
+
+ torch.testing.assert_close(
+ pooled[0],
+ ref_one(hs_list[0], prompt_lens[0], token_ids[0]),
+ rtol=1e-4,
+ atol=1e-4,
+ )
+ torch.testing.assert_close(
+ pooled[1],
+ ref_one(hs_list[1], prompt_lens[1], token_ids[1]),
+ rtol=1e-4,
+ atol=1e-4,
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_token_classification.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_token_classification.py
new file mode 100644
index 0000000000000000000000000000000000000000..63bee1f10528dec7f15a64595c55481e0383a483
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_token_classification.py
@@ -0,0 +1,198 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+import torch
+from transformers import AutoModelForTokenClassification
+
+from tests.models.registry import HF_EXAMPLE_MODELS
+from tests.models.utils import softmax
+from vllm.platforms import current_platform
+from vllm.utils.torch_utils import set_random_seed
+
+
+@pytest.fixture(autouse=True)
+def seed_everything():
+ """Seed all random number generators for reproducibility."""
+ seed = 0
+ set_random_seed(seed)
+ torch.backends.cudnn.deterministic = True
+ torch.backends.cudnn.benchmark = False
+ yield
+
+
+@pytest.mark.parametrize(
+ "model",
+ [
+ "boltuix/NeuroBERT-NER",
+ ],
+)
+# The float32 is required for this tiny model to pass the test.
+@pytest.mark.parametrize("dtype", ["float"])
+@torch.inference_mode
+def test_bert_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
+ vllm_outputs = vllm_model.token_classify(example_prompts)
+
+ # Use eager attention on ROCm to avoid HF Transformers flash attention
+ # accuracy issues: https://github.com/vllm-project/vllm/issues/30167
+ hf_model_kwargs = {}
+ if current_platform.is_rocm():
+ hf_model_kwargs["attn_implementation"] = "eager"
+
+ with hf_runner(
+ model,
+ dtype=dtype,
+ auto_cls=AutoModelForTokenClassification,
+ model_kwargs=hf_model_kwargs,
+ ) as hf_model:
+ tokenizer = hf_model.tokenizer
+ hf_outputs = []
+ for prompt in example_prompts:
+ inputs = tokenizer([prompt], return_tensors="pt")
+ inputs = hf_model.wrap_device(inputs)
+ output = hf_model.model(**inputs)
+ hf_outputs.append(softmax(output.logits[0]))
+
+ # check logits difference
+ for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+ hf_output = hf_output.detach().clone().cpu().float()
+ vllm_output = vllm_output.detach().clone().cpu().float()
+ torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
+
+
+@pytest.mark.parametrize("model", ["disham993/electrical-ner-ModernBERT-base"])
+@pytest.mark.parametrize("dtype", ["float"])
+@pytest.mark.flaky(reruns=3)
+@torch.inference_mode
+def test_modernbert_models(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ # NOTE: https://github.com/vllm-project/vllm/pull/32403
+ # `disham993/electrical-ner-ModernBERT-base` is a randomly initialized
+ # model, which can cause numerical precision variance and edge cases.
+ # We use @flaky(reruns=3) to mitigate intermittent failures.
+ print(
+ f"\n[NOTE] Testing {model} (randomly initialized weights) - "
+ "flaky tolerance enabled due to numerical precision variance."
+ )
+
+ with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
+ vllm_outputs = vllm_model.token_classify(example_prompts)
+
+ # Use eager attention on ROCm to avoid HF Transformers flash attention
+ # accuracy issues: https://github.com/vllm-project/vllm/issues/30167
+ hf_model_kwargs = {}
+ if current_platform.is_rocm():
+ hf_model_kwargs["attn_implementation"] = "eager"
+
+ with hf_runner(
+ model,
+ dtype=dtype,
+ auto_cls=AutoModelForTokenClassification,
+ model_kwargs=hf_model_kwargs,
+ ) as hf_model:
+ tokenizer = hf_model.tokenizer
+ hf_outputs = []
+ for prompt in example_prompts:
+ inputs = tokenizer([prompt], return_tensors="pt")
+ inputs = hf_model.wrap_device(inputs)
+ output = hf_model.model(**inputs)
+ hf_outputs.append(softmax(output.logits[0]))
+
+ # check logits difference
+ for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+ hf_output = hf_output.detach().clone().cpu().float()
+ vllm_output = vllm_output.detach().clone().cpu().float()
+ torch.testing.assert_close(hf_output, vllm_output, atol=3.2e-2, rtol=1e-3)
+
+
+PRIVACY_FILTER_PROMPTS = [
+ "My name is Harry Potter.",
+ "Email me at harry.potter@hogwarts.edu.",
+ "Call me on +44 20 7946 0958 tomorrow.",
+ "My account number is 12345678 and the API key is sk-live-abc123def456.",
+ "I live at 4 Privet Drive, Little Whinging.",
+ "Visit https://example.com/profile/harry for more info.",
+ "We met on 12 January 2024.",
+]
+
+
+@pytest.mark.parametrize("model", ["openai/privacy-filter"])
+@pytest.mark.parametrize("dtype", ["bfloat16"])
+@torch.inference_mode
+def test_openai_privacy_filter(
+ hf_runner,
+ vllm_runner,
+ model: str,
+ dtype: str,
+) -> None:
+ model_info = HF_EXAMPLE_MODELS.find_hf_info(model)
+ model_info.check_transformers_version(on_fail="skip")
+
+ with vllm_runner(model, max_model_len=None, dtype=dtype) as vllm_model:
+ vllm_outputs = vllm_model.token_classify(PRIVACY_FILTER_PROMPTS)
+
+ hf_model_kwargs = {}
+ if current_platform.is_rocm():
+ hf_model_kwargs["attn_implementation"] = "eager"
+
+ with hf_runner(
+ model,
+ dtype=dtype,
+ auto_cls=AutoModelForTokenClassification,
+ model_kwargs=hf_model_kwargs,
+ ) as hf_model:
+ tokenizer = hf_model.tokenizer
+ hf_outputs = []
+ for prompt in PRIVACY_FILTER_PROMPTS:
+ inputs = tokenizer([prompt], return_tensors="pt")
+ inputs = hf_model.wrap_device(inputs)
+ output = hf_model.model(**inputs)
+ hf_outputs.append(softmax(output.logits[0]))
+
+ for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+ hf_output = hf_output.detach().clone().cpu().float()
+ vllm_output = vllm_output.detach().clone().cpu().float()
+ torch.testing.assert_close(hf_output, vllm_output, atol=0.1, rtol=1e-2)
+
+
+@pytest.mark.parametrize("model", ["bd2lcco/Qwen3-0.6B-finetuned"])
+@pytest.mark.parametrize("dtype", ["float"])
+@torch.inference_mode
+def test_auto_conversion(
+ hf_runner,
+ vllm_runner,
+ example_prompts,
+ model: str,
+ dtype: str,
+) -> None:
+ with vllm_runner(model, max_model_len=1024, dtype=dtype) as vllm_model:
+ vllm_outputs = vllm_model.token_classify(example_prompts)
+
+ with hf_runner(
+ model, dtype=dtype, auto_cls=AutoModelForTokenClassification
+ ) as hf_model:
+ tokenizer = hf_model.tokenizer
+ hf_outputs = []
+ for prompt in example_prompts:
+ inputs = tokenizer([prompt], return_tensors="pt")
+ inputs = hf_model.wrap_device(inputs)
+ output = hf_model.model(**inputs)
+ hf_outputs.append(softmax(output.logits[0]))
+
+ # check logits difference
+ for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+ hf_output = hf_output.detach().clone().cpu().float()
+ vllm_output = vllm_output.detach().clone().cpu().float()
+ assert torch.allclose(hf_output, vllm_output, atol=1e-2)
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling/test_truncation_control.py b/asian_ai/performance/vLLM/tests/models/language/pooling/test_truncation_control.py
new file mode 100644
index 0000000000000000000000000000000000000000..1e54288c969bd8a3f026f80aae1fd600cd561ef8
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling/test_truncation_control.py
@@ -0,0 +1,79 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+
+MODEL_NAME = "sentence-transformers/all-MiniLM-L12-v2"
+max_model_len = 128
+
+input_str = """Immerse yourself in the enchanting chronicle of calculus, a
+mathematical domain that has radically transformed our comprehension of
+change and motion. Despite its roots in ancient civilizations, the
+formal birth of calculus predominantly occurred in the 17th century,
+primarily under the influential guidance of Sir Isaac Newton and Gottfried
+Wilhelm Leibniz. The earliest traces of calculus concepts are found in
+ancient Greek mathematics,most notably in the works of Eudoxus and
+Archimedes, around 300 BCE. They utilized the 'method of exhaustion'—a
+technique for computing areas and volumes through the use of finite sums.
+This methodology laid crucial foundational work for integral calculus.
+In the 17th century, both Newton and Leibniz independently pioneered
+calculus, each contributing unique perspectives that would shape this new
+field."""
+
+
+def test_smaller_truncation_size(
+ vllm_runner, model_name=MODEL_NAME, input_str=input_str
+):
+ truncate_prompt_tokens = 10
+
+ with vllm_runner(
+ model_name, runner="pooling", max_model_len=max_model_len
+ ) as vllm_model:
+ vllm_output = vllm_model.llm.embed(
+ input_str,
+ tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
+ )
+
+ prompt_tokens = vllm_output[0].prompt_token_ids
+
+ assert len(prompt_tokens) == truncate_prompt_tokens
+
+
+def test_max_truncation_size(vllm_runner, model_name=MODEL_NAME, input_str=input_str):
+ truncate_prompt_tokens = -1
+
+ with vllm_runner(
+ model_name, runner="pooling", max_model_len=max_model_len
+ ) as vllm_model:
+ vllm_output = vllm_model.llm.embed(
+ input_str,
+ tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
+ )
+
+ prompt_tokens = vllm_output[0].prompt_token_ids
+
+ assert len(prompt_tokens) == max_model_len
+
+
+def test_bigger_truncation_size(
+ vllm_runner, model_name=MODEL_NAME, input_str=input_str
+):
+ truncate_prompt_tokens = max_model_len + 1
+
+ with (
+ pytest.raises(ValueError),
+ vllm_runner(
+ model_name, runner="pooling", max_model_len=max_model_len
+ ) as vllm_model,
+ ):
+ llm_output = vllm_model.llm.embed(
+ input_str,
+ tokenization_kwargs=dict(truncate_prompt_tokens=truncate_prompt_tokens),
+ )
+
+ assert (
+ llm_output
+ == f"""truncate_prompt_tokens value
+ ({truncate_prompt_tokens}) is greater than
+ max_model_len ({max_model_len}). Please, select
+ a smaller truncation size."""
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/__init__.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/mteb_embed_utils.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/mteb_embed_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..17a56d7883b8eeee2e3a3be88ecf740475e752bc
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/mteb_embed_utils.py
@@ -0,0 +1,258 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import mteb
+import numpy as np
+import torch
+from mteb.models import ModelMeta
+from mteb.types import Array
+from torch.utils.data import DataLoader
+
+import tests.ci_envs as ci_envs
+from tests.models.utils import (
+ EmbedModelInfo,
+ check_embeddings_close,
+ get_vllm_extra_kwargs,
+)
+
+# Most embedding models on the STS12 task (See #17175):
+# - Model implementation and minor changes in tensor dtype
+# results in differences less than 1e-4
+# - Different model results in differences more than 1e-3
+# 5e-4 is a good tolerance threshold
+MTEB_EMBED_TASKS = ["STS12"]
+MTEB_EMBED_TOL = 5e-4
+
+
+_empty_model_meta = ModelMeta(
+ loader=None,
+ name="vllm/model",
+ revision="1",
+ release_date=None,
+ languages=None,
+ framework=[],
+ similarity_fn_name=None,
+ n_parameters=None,
+ memory_usage_mb=None,
+ max_tokens=None,
+ embed_dim=None,
+ license=None,
+ open_weights=None,
+ public_training_code=None,
+ public_training_data=None,
+ use_instructions=None,
+ training_datasets=None,
+ modalities=["text"], # 'image' can be added to evaluate multimodal models
+)
+
+
+class MtebEmbedMixin(mteb.EncoderProtocol):
+ mteb_model_meta = _empty_model_meta
+
+ def similarity(
+ self,
+ embeddings1: np.ndarray,
+ embeddings2: np.ndarray,
+ ) -> np.ndarray:
+ # Cosine similarity
+ norm1 = np.linalg.norm(embeddings1, axis=1, keepdims=True)
+ norm2 = np.linalg.norm(embeddings2, axis=1, keepdims=True)
+ sim = np.dot(embeddings1, embeddings2.T) / (norm1 * norm2.T)
+ return sim
+
+ def similarity_pairwise(
+ self,
+ embeddings1: Array,
+ embeddings2: Array,
+ ) -> Array:
+ # Cosine similarity
+ norm1 = np.linalg.norm(embeddings1, axis=1, keepdims=True)
+ norm2 = np.linalg.norm(embeddings2, axis=1, keepdims=True)
+ sim = np.sum(embeddings1 * embeddings2, axis=1) / (
+ norm1.flatten() * norm2.flatten()
+ )
+ return sim
+
+
+class HfMtebEncoder(MtebEmbedMixin):
+ def __init__(self, model):
+ self.model = model
+
+ def encode(
+ self,
+ inputs: DataLoader[mteb.types.BatchedInput],
+ *args,
+ **kwargs,
+ ) -> np.ndarray:
+ sentences = [text for batch in inputs for text in batch["text"]]
+ return self.model.encode(sentences)
+
+
+class VllmMtebEncoder(MtebEmbedMixin):
+ def __init__(self, vllm_model, prompt_prefix: str | None = None):
+ self.llm = vllm_model
+ self.rng = np.random.default_rng(seed=42)
+ self.prompt_prefix = prompt_prefix
+
+ def encode(
+ self,
+ inputs: DataLoader[mteb.types.BatchedInput],
+ *args,
+ **kwargs,
+ ) -> np.ndarray:
+ # Hoping to discover potential scheduling
+ # issues by randomizing the order.
+ sentences = [
+ self.prompt_prefix + text if self.prompt_prefix else text
+ for batch in inputs
+ for text in batch["text"]
+ ]
+ r = self.rng.permutation(len(sentences))
+ sentences = [sentences[i] for i in r]
+ outputs = self.llm.embed(sentences, use_tqdm=False)
+ embeds = np.array(outputs)
+ embeds = embeds[np.argsort(r)]
+ return embeds
+
+
+class OpenAIClientMtebEncoder(MtebEmbedMixin):
+ def __init__(self, model_name: str, client):
+ self.model_name = model_name
+ self.client = client
+ self.rng = np.random.default_rng(seed=42)
+
+ def encode(
+ self,
+ inputs: DataLoader[mteb.types.BatchedInput],
+ *args,
+ **kwargs,
+ ) -> np.ndarray:
+ # Hoping to discover potential scheduling
+ # issues by randomizing the order.
+ sentences = [text for batch in inputs for text in batch["text"]]
+ r = self.rng.permutation(len(sentences))
+ sentences = [sentences[i] for i in r]
+
+ embeddings = self.client.embeddings.create(
+ model=self.model_name, input=sentences
+ )
+ outputs = [d.embedding for d in embeddings.data]
+ embeds = np.array(outputs)
+ embeds = embeds[np.argsort(r)]
+ return embeds
+
+
+def run_mteb_embed_task(encoder: mteb.EncoderProtocol, tasks):
+ tasks = mteb.get_tasks(tasks=tasks)
+ results = mteb.evaluate(
+ encoder,
+ tasks,
+ cache=None,
+ show_progress_bar=False,
+ )
+
+ main_score = results[0].scores["test"][0]["main_score"]
+ return main_score
+
+
+def mteb_test_embed_models(
+ hf_runner,
+ vllm_runner,
+ model_info: EmbedModelInfo,
+ vllm_extra_kwargs=None,
+ hf_model_callback=None,
+ atol=MTEB_EMBED_TOL,
+ prompt_prefix: str | None = None,
+):
+ vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
+
+ # Test embed_dims, isnan and whether to use normalize
+ example_prompts = ["The chef prepared a delicious meal." * 1000]
+
+ with vllm_runner(
+ model_info.name,
+ revision=model_info.revision,
+ runner="pooling",
+ max_model_len=model_info.max_model_len,
+ **vllm_extra_kwargs,
+ ) as vllm_model:
+ model_config = vllm_model.llm.llm_engine.model_config
+
+ # Confirm whether vllm is using the correct architecture
+ if model_info.architecture:
+ assert model_info.architecture in model_config.architectures
+
+ # Confirm whether the important configs in model_config are correct.
+ pooler_config = model_config.pooler_config
+ if model_info.seq_pooling_type is not None:
+ assert pooler_config.seq_pooling_type == model_info.seq_pooling_type
+ if model_info.tok_pooling_type is not None:
+ assert pooler_config.tok_pooling_type == model_info.tok_pooling_type
+ if model_info.attn_type is not None:
+ assert model_config.attn_type == model_info.attn_type
+ if model_info.is_prefix_caching_supported is not None:
+ assert (
+ model_config.is_prefix_caching_supported
+ == model_info.is_prefix_caching_supported
+ )
+ if model_info.is_chunked_prefill_supported is not None:
+ assert (
+ model_config.is_chunked_prefill_supported
+ == model_info.is_chunked_prefill_supported
+ )
+
+ vllm_main_score = run_mteb_embed_task(
+ VllmMtebEncoder(vllm_model, prompt_prefix=prompt_prefix), MTEB_EMBED_TASKS
+ )
+ vllm_dtype = vllm_model.llm.llm_engine.model_config.dtype
+ head_dtype = model_config.head_dtype
+
+ # Test embedding_size, isnan and whether to use normalize
+ vllm_outputs = vllm_model.embed(
+ example_prompts,
+ tokenization_kwargs=dict(truncate_prompt_tokens=-1),
+ )
+ outputs_tensor = torch.tensor(vllm_outputs)
+ assert not torch.any(torch.isnan(outputs_tensor))
+ embedding_size = model_config.embedding_size
+ assert torch.tensor(vllm_outputs).shape[-1] == embedding_size
+
+ # Accelerate mteb test by setting
+ # SentenceTransformers mteb score to a constant
+ if model_info.mteb_score is None:
+ with hf_runner(
+ model_info.name,
+ revision=model_info.revision,
+ is_sentence_transformer=True,
+ dtype=ci_envs.VLLM_CI_HF_DTYPE or model_info.hf_dtype,
+ ) as hf_model:
+ # e.g. setting default parameters for the encode method of hf_runner
+ if hf_model_callback is not None:
+ hf_model_callback(hf_model)
+
+ st_main_score = run_mteb_embed_task(
+ HfMtebEncoder(hf_model), MTEB_EMBED_TASKS
+ )
+ st_dtype = next(hf_model.model.parameters()).dtype
+
+ # Check embeddings close to hf outputs
+ hf_outputs = hf_model.encode(example_prompts)
+ check_embeddings_close(
+ embeddings_0_lst=hf_outputs,
+ embeddings_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ tol=1e-2,
+ )
+ else:
+ st_main_score = model_info.mteb_score
+ st_dtype = "Constant"
+
+ print("Model:", model_info.name)
+ print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_main_score)
+ print("SentenceTransformers:", st_dtype, st_main_score)
+ print("Difference:", st_main_score - vllm_main_score)
+
+ # We are not concerned that the vllm mteb results are better
+ # than SentenceTransformers, so we only perform one-sided testing.
+ assert st_main_score - vllm_main_score < atol
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/mteb_score_utils.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/mteb_score_utils.py
new file mode 100644
index 0000000000000000000000000000000000000000..ddaf411b8512ccaa606744245805598bb3090194
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/mteb_score_utils.py
@@ -0,0 +1,311 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import tempfile
+from pathlib import Path
+from typing import Any
+
+import mteb
+import numpy as np
+import requests
+import torch
+from mteb.models import ModelMeta
+from torch.utils.data import DataLoader
+
+from tests.conftest import HfRunner
+from tests.models.utils import (
+ RerankModelInfo,
+ get_vllm_extra_kwargs,
+)
+
+# See #19344
+MTEB_RERANK_TASKS = ["NFCorpus"]
+MTEB_RERANK_LANGS = ["eng"]
+MTEB_RERANK_TOL = 2e-3
+
+template_home = (
+ Path(__file__).parent.parent.parent.parent.parent
+ / "examples/pooling/score/template"
+)
+
+_empty_model_meta = ModelMeta(
+ loader=None,
+ name="vllm/model",
+ revision="1",
+ release_date=None,
+ languages=None,
+ framework=[],
+ similarity_fn_name=None,
+ n_parameters=None,
+ memory_usage_mb=None,
+ max_tokens=None,
+ embed_dim=None,
+ license=None,
+ open_weights=None,
+ public_training_code=None,
+ public_training_data=None,
+ use_instructions=None,
+ training_datasets=None,
+ modalities=["text"], # 'image' can be added to evaluate multimodal models
+)
+
+
+class MtebCrossEncoderMixin(mteb.CrossEncoderProtocol):
+ mteb_model_meta = _empty_model_meta
+
+
+class VllmMtebCrossEncoder(MtebCrossEncoderMixin):
+ def __init__(self, vllm_model):
+ self.llm = vllm_model
+ self.rng = np.random.default_rng(seed=42)
+ self.chat_template: str | None = getattr(vllm_model, "chat_template", None)
+
+ def predict(
+ self,
+ inputs1: DataLoader[mteb.types.BatchedInput],
+ inputs2: DataLoader[mteb.types.BatchedInput],
+ *args,
+ **kwargs,
+ ) -> np.ndarray:
+ queries = [text for batch in inputs1 for text in batch["text"]]
+ corpus = [text for batch in inputs2 for text in batch["text"]]
+
+ # Hoping to discover potential scheduling
+ # issues by randomizing the order.
+ r = self.rng.permutation(len(queries))
+ queries = [queries[i] for i in r]
+ corpus = [corpus[i] for i in r]
+
+ outputs = self.llm.score(
+ queries,
+ corpus,
+ use_tqdm=False,
+ chat_template=self.chat_template,
+ tokenization_kwargs={"truncate_prompt_tokens": -1},
+ )
+ scores = np.array(outputs)
+ scores = scores[np.argsort(r)]
+ return scores
+
+
+class ScoreClientMtebEncoder(MtebCrossEncoderMixin):
+ mteb_model_meta = _empty_model_meta
+
+ def __init__(self, model_name: str, url):
+ self.model_name = model_name
+ self.url = url
+
+ def predict(
+ self,
+ inputs1: DataLoader[mteb.types.BatchedInput],
+ inputs2: DataLoader[mteb.types.BatchedInput],
+ *args,
+ **kwargs,
+ ) -> np.ndarray:
+ queries = [text for batch in inputs1 for text in batch["text"]]
+ full_corpus = [text for batch in inputs2 for text in batch["text"]]
+
+ outputs = []
+ for query, corpus in zip(queries, full_corpus):
+ outputs.append(self.get_score(query, corpus))
+
+ scores = np.array(outputs)
+ return scores
+
+ def get_score(self, query, corpus):
+ response = requests.post(
+ self.url,
+ json={
+ "model": self.model_name,
+ "queries": query,
+ "documents": corpus,
+ "truncate_prompt_tokens": -1,
+ },
+ ).json()
+ return response["data"][0]["score"]
+
+
+class RerankClientMtebEncoder(ScoreClientMtebEncoder):
+ def get_score(self, query, corpus):
+ response = requests.post(
+ self.url,
+ json={
+ "model": self.model_name,
+ "query": query,
+ "documents": [corpus],
+ "truncate_prompt_tokens": -1,
+ },
+ ).json()
+ return response["results"][0]["relevance_score"]
+
+
+class HFMtebCrossEncoder(MtebCrossEncoderMixin, HfRunner):
+ chat_template: str | None = None
+
+ def __init__(self, model_name: str, dtype: str = "auto", **kwargs: Any) -> None:
+ HfRunner.__init__(
+ self, model_name=model_name, is_cross_encoder=True, dtype=dtype, **kwargs
+ )
+
+ @torch.no_grad
+ def predict(
+ self,
+ inputs1: DataLoader[mteb.types.BatchedInput],
+ inputs2: DataLoader[mteb.types.BatchedInput],
+ *args,
+ **kwargs,
+ ) -> np.ndarray:
+ queries = [text for batch in inputs1 for text in batch["text"]]
+ corpus = [text for batch in inputs2 for text in batch["text"]]
+
+ if self.chat_template is not None:
+ tokenizer = self.model.tokenizer
+ prompts = []
+ for query, document in zip(queries, corpus):
+ conversation = [
+ {"role": "query", "content": query},
+ {"role": "document", "content": document},
+ ]
+
+ prompt = tokenizer.apply_chat_template(
+ conversation=conversation,
+ tools=None,
+ chat_template=self.chat_template,
+ tokenize=False,
+ )
+ prompts.append(prompt)
+ outputs_list = HfRunner.classify(self, prompts)
+ scores = np.array(outputs_list).squeeze(-1)
+ return scores
+ else:
+ prompts = list(zip(queries, corpus))
+ outputs_tensor = HfRunner.predict(self, prompts, show_progress_bar=False)
+ return outputs_tensor.cpu().numpy()
+
+
+def run_mteb_rerank(cross_encoder: mteb.CrossEncoderProtocol, tasks, languages):
+ with tempfile.TemporaryDirectory() as prediction_folder:
+ bm25s = mteb.get_model("bm25s")
+ eval_splits = ["test"]
+
+ mteb_tasks: list[mteb.abstasks.AbsTaskRetrieval] = mteb.get_tasks(
+ tasks=tasks, languages=languages, eval_splits=eval_splits
+ )
+ for task in mteb_tasks:
+ if not task.data_loaded:
+ task.load_data()
+
+ mteb.evaluate(
+ bm25s,
+ mteb_tasks,
+ prediction_folder=prediction_folder,
+ show_progress_bar=False,
+ # don't save results for test runs
+ cache=None,
+ overwrite_strategy="always",
+ )
+
+ second_stage_tasks = []
+ for task in mteb_tasks:
+ second_stage_tasks.append(
+ task.convert_to_reranking(
+ prediction_folder,
+ top_k=10,
+ )
+ )
+
+ results = mteb.evaluate(
+ cross_encoder,
+ second_stage_tasks,
+ show_progress_bar=False,
+ cache=None,
+ )
+ main_score = results[0].scores["test"][0]["main_score"]
+ return main_score
+
+
+def mteb_test_rerank_models(
+ vllm_runner,
+ model_info: RerankModelInfo,
+ hf_runner=HFMtebCrossEncoder,
+ vllm_extra_kwargs=None,
+ vllm_mteb_encoder=VllmMtebCrossEncoder,
+ atol=MTEB_RERANK_TOL,
+):
+ vllm_extra_kwargs = get_vllm_extra_kwargs(model_info, vllm_extra_kwargs)
+
+ # Maybe load chat_template.
+ chat_template: str | None = None
+ if model_info.chat_template_name is not None:
+ chat_template = (template_home / model_info.chat_template_name).read_text()
+
+ with vllm_runner(
+ model_info.name,
+ revision=model_info.revision,
+ runner="pooling",
+ max_model_len=None,
+ max_num_seqs=8,
+ **vllm_extra_kwargs,
+ ) as vllm_model:
+ model_config = vllm_model.llm.llm_engine.model_config
+ vllm_model.chat_template = chat_template
+
+ # Confirm whether vllm is using the correct architecture
+ if model_info.architecture:
+ assert model_info.architecture in model_config.architectures
+
+ # Score API is only enabled for num_labels == 1
+ assert model_config.hf_config.num_labels == 1
+
+ # Confirm whether the important configs in model_config are correct.
+ pooler_config = model_config.pooler_config
+ if model_info.seq_pooling_type is not None:
+ assert pooler_config.seq_pooling_type == model_info.seq_pooling_type
+ if model_info.tok_pooling_type is not None:
+ assert pooler_config.tok_pooling_type == model_info.tok_pooling_type
+ if model_info.attn_type is not None:
+ assert model_config.attn_type == model_info.attn_type
+ if model_info.is_prefix_caching_supported is not None:
+ assert (
+ model_config.is_prefix_caching_supported
+ == model_info.is_prefix_caching_supported
+ )
+ if model_info.is_chunked_prefill_supported is not None:
+ assert (
+ model_config.is_chunked_prefill_supported
+ == model_info.is_chunked_prefill_supported
+ )
+
+ vllm_main_score = run_mteb_rerank(
+ vllm_mteb_encoder(vllm_model),
+ tasks=MTEB_RERANK_TASKS,
+ languages=MTEB_RERANK_LANGS,
+ )
+ vllm_dtype = model_config.dtype
+ head_dtype = model_config.head_dtype
+
+ # Accelerate mteb test by setting
+ # SentenceTransformers mteb score to a constant
+ if model_info.mteb_score is None:
+ with hf_runner(
+ model_info.name, revision=model_info.revision, dtype=model_info.hf_dtype
+ ) as hf_model:
+ hf_model.chat_template = chat_template
+ st_main_score = run_mteb_rerank(
+ hf_model,
+ tasks=MTEB_RERANK_TASKS,
+ languages=MTEB_RERANK_LANGS,
+ )
+ st_dtype = next(hf_model.model.model.parameters()).dtype
+ else:
+ st_main_score = model_info.mteb_score
+ st_dtype = "Constant"
+
+ print("Model:", model_info.name)
+ print("VLLM:", f"dtype:{vllm_dtype}", f"head_dtype:{head_dtype}", vllm_main_score)
+ print("SentenceTransformers:", st_dtype, st_main_score)
+ print("Difference:", st_main_score - vllm_main_score)
+
+ # We are not concerned that the vllm mteb results are better
+ # than SentenceTransformers, so we only perform one-sided testing.
+ assert st_main_score - vllm_main_score < atol
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_baai.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_baai.py
new file mode 100644
index 0000000000000000000000000000000000000000..11b910d14527fa42255be3cd8597208abe341285
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_baai.py
@@ -0,0 +1,118 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+
+from tests.models.language.pooling.embed_utils import correctness_test_embed_models
+from tests.models.utils import (
+ EmbedModelInfo,
+ RerankModelInfo,
+)
+
+from .mteb_embed_utils import mteb_test_embed_models
+from .mteb_score_utils import mteb_test_rerank_models
+
+MODELS = [
+ ########## BertModel
+ EmbedModelInfo(
+ "BAAI/bge-base-en",
+ architecture="BertModel",
+ mteb_score=0.779336792,
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ EmbedModelInfo("BAAI/bge-base-zh", architecture="BertModel", enable_test=False),
+ EmbedModelInfo("BAAI/bge-small-en", architecture="BertModel", enable_test=False),
+ EmbedModelInfo("BAAI/bge-small-zh", architecture="BertModel", enable_test=False),
+ EmbedModelInfo("BAAI/bge-large-en", architecture="BertModel", enable_test=False),
+ EmbedModelInfo("BAAI/bge-large-zh", architecture="BertModel", enable_test=False),
+ EmbedModelInfo(
+ "BAAI/bge-large-zh-noinstruct", architecture="BertModel", enable_test=False
+ ),
+ EmbedModelInfo(
+ "BAAI/bge-base-en-v1.5", architecture="BertModel", enable_test=False
+ ),
+ EmbedModelInfo(
+ "BAAI/bge-base-zh-v1.5", architecture="BertModel", enable_test=False
+ ),
+ EmbedModelInfo(
+ "BAAI/bge-small-en-v1.5", architecture="BertModel", enable_test=False
+ ),
+ EmbedModelInfo(
+ "BAAI/bge-small-zh-v1.5", architecture="BertModel", enable_test=False
+ ),
+ EmbedModelInfo(
+ "BAAI/bge-large-en-v1.5", architecture="BertModel", enable_test=False
+ ),
+ EmbedModelInfo(
+ "BAAI/bge-large-zh-v1.5", architecture="BertModel", enable_test=False
+ ),
+ ########## XLMRobertaModel
+ EmbedModelInfo(
+ "BAAI/bge-m3",
+ architecture="XLMRobertaModel",
+ mteb_score=0.787343078,
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ ########## Qwen2Model
+ EmbedModelInfo(
+ "BAAI/bge-code-v1",
+ architecture="Qwen2Model",
+ mteb_score=0.75724465,
+ seq_pooling_type="LAST",
+ attn_type="decoder",
+ is_prefix_caching_supported=True,
+ is_chunked_prefill_supported=True,
+ # Skip: model's custom tokenizer on HF hub is incompatible with
+ # transformers v5 (sets attrs before super().__init__, triggering
+ # AttributeError on 'verbose' in __getattr__).
+ enable_test=False,
+ ),
+]
+
+RERANK_MODELS = [
+ ########## XLMRobertaForSequenceClassification
+ RerankModelInfo(
+ "BAAI/bge-reranker-base",
+ architecture="XLMRobertaForSequenceClassification",
+ mteb_score=0.32398,
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ RerankModelInfo(
+ "BAAI/bge-reranker-large",
+ architecture="XLMRobertaForSequenceClassification",
+ enable_test=False,
+ ),
+ RerankModelInfo(
+ "BAAI/bge-reranker-v2-m3",
+ architecture="XLMRobertaForSequenceClassification",
+ enable_test=False,
+ ),
+]
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
+ mteb_test_embed_models(hf_runner, vllm_runner, model_info)
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_embed_models_correctness(
+ hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
+) -> None:
+ correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
+
+
+@pytest.mark.parametrize("model_info", RERANK_MODELS)
+def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
+ mteb_test_rerank_models(vllm_runner, model_info)
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py
new file mode 100644
index 0000000000000000000000000000000000000000..a5cdcc4855d3bad9ca76595b872164dabe2c5628
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_bge_reranker_v2_gemma.py
@@ -0,0 +1,136 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from typing import Any
+
+import mteb
+import numpy as np
+import pytest
+import torch
+from torch.utils.data import DataLoader
+
+from tests.conftest import HfRunner
+from tests.models.utils import RerankModelInfo
+
+from .mteb_score_utils import (
+ MtebCrossEncoderMixin,
+ mteb_test_rerank_models,
+)
+
+RERANK_MODELS = [
+ RerankModelInfo(
+ "BAAI/bge-reranker-v2-gemma",
+ architecture="GemmaForSequenceClassification",
+ hf_overrides={
+ "architectures": ["GemmaForSequenceClassification"],
+ "classifier_from_token": ["Yes"],
+ "method": "no_post_processing",
+ },
+ mteb_score=0.33757,
+ seq_pooling_type="LAST",
+ attn_type="decoder",
+ is_prefix_caching_supported=True,
+ is_chunked_prefill_supported=True,
+ chat_template_name="bge-reranker-v2-gemma.jinja",
+ ),
+]
+
+PROMPT = "Given a query A and a passage B, determine whether the passage contains an answer to the query by providing a prediction of either 'Yes' or 'No'." # noqa: E501
+
+
+class GemmaRerankerHfRunner(MtebCrossEncoderMixin, HfRunner):
+ def __init__(
+ self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
+ ) -> None:
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+
+ HfRunner.__init__(
+ self,
+ model_name=model_name,
+ auto_cls=AutoModelForCausalLM,
+ dtype=dtype,
+ **kwargs,
+ )
+
+ self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
+ self.yes_loc = self.tokenizer.convert_tokens_to_ids("Yes")
+
+ @torch.no_grad
+ def predict(
+ self,
+ inputs1: DataLoader[mteb.types.BatchedInput],
+ inputs2: DataLoader[mteb.types.BatchedInput],
+ *args,
+ **kwargs,
+ ) -> np.ndarray:
+ queries = [text for batch in inputs1 for text in batch["text"]]
+ corpus = [text for batch in inputs2 for text in batch["text"]]
+
+ def get_inputs(pairs, tokenizer, prompt=None):
+ if prompt is None:
+ prompt = PROMPT
+
+ sep = "\n"
+ prompt_inputs = tokenizer(
+ prompt, return_tensors=None, add_special_tokens=False
+ )["input_ids"]
+ sep_inputs = tokenizer(sep, return_tensors=None, add_special_tokens=False)[
+ "input_ids"
+ ]
+ inputs = []
+ for query, passage in pairs:
+ query_inputs = tokenizer(
+ f"A: {query}",
+ return_tensors=None,
+ add_special_tokens=False,
+ truncation=True,
+ )
+ passage_inputs = tokenizer(
+ f"B: {passage}",
+ return_tensors=None,
+ add_special_tokens=False,
+ truncation=True,
+ )
+ item = tokenizer.prepare_for_model(
+ [tokenizer.bos_token_id] + query_inputs["input_ids"],
+ sep_inputs + passage_inputs["input_ids"],
+ truncation="only_second",
+ padding=False,
+ return_attention_mask=False,
+ return_token_type_ids=False,
+ add_special_tokens=False,
+ )
+ item["input_ids"] = item["input_ids"] + sep_inputs + prompt_inputs
+ item["attention_mask"] = [1] * len(item["input_ids"])
+ inputs.append(item)
+ return tokenizer.pad(
+ inputs,
+ padding=True,
+ return_tensors="pt",
+ )
+
+ scores = []
+ for query, document in zip(queries, corpus):
+ pairs = [(query, document)]
+ inputs = get_inputs(pairs, self.tokenizer)
+ inputs = inputs.to(self.model.device)
+ _n_tokens = inputs["input_ids"].shape[1]
+ logits = self.model(**inputs, return_dict=True).logits
+ _scores = (
+ logits[:, -1, self.yes_loc]
+ .view(
+ -1,
+ )
+ .float()
+ .sigmoid()
+ )
+ scores.append(_scores[0].item())
+ return torch.Tensor(scores)
+
+
+@pytest.mark.parametrize("model_info", RERANK_MODELS)
+def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
+ mteb_test_rerank_models(
+ vllm_runner,
+ model_info,
+ hf_runner=GemmaRerankerHfRunner,
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_cross_encoder.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_cross_encoder.py
new file mode 100644
index 0000000000000000000000000000000000000000..147f3aaaf462377379591e66e45a7bc1b158477f
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_cross_encoder.py
@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+
+from tests.models.utils import (
+ RerankModelInfo,
+)
+
+from .mteb_score_utils import mteb_test_rerank_models
+
+RERANK_MODELS = [
+ RerankModelInfo(
+ "cross-encoder/ms-marco-TinyBERT-L-2-v2",
+ architecture="BertForSequenceClassification",
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ mteb_score=0.32898,
+ ),
+ RerankModelInfo(
+ "tomaarsen/Qwen3-Reranker-0.6B-seq-cls",
+ architecture="Qwen3ForSequenceClassification",
+ seq_pooling_type="LAST",
+ attn_type="decoder",
+ is_prefix_caching_supported=True,
+ is_chunked_prefill_supported=True,
+ chat_template_name="qwen3_reranker.jinja",
+ mteb_score=0.33459,
+ ),
+]
+
+
+@pytest.mark.parametrize("model_info", RERANK_MODELS)
+def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
+ mteb_test_rerank_models(vllm_runner, model_info)
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_gte.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_gte.py
new file mode 100644
index 0000000000000000000000000000000000000000..9015d1211bc2a192d1424232f88a93138df7ee4d
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_gte.py
@@ -0,0 +1,152 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from tests.models.language.pooling.embed_utils import correctness_test_embed_models
+from tests.models.utils import (
+ EmbedModelInfo,
+ RerankModelInfo,
+)
+from vllm.platforms import current_platform
+
+from .mteb_embed_utils import mteb_test_embed_models
+from .mteb_score_utils import mteb_test_rerank_models
+
+MODELS = [
+ ########## BertModel
+ EmbedModelInfo(
+ "thenlper/gte-large",
+ mteb_score=0.76807651,
+ architecture="BertModel",
+ seq_pooling_type="MEAN",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ EmbedModelInfo("thenlper/gte-base", architecture="BertModel", enable_test=False),
+ EmbedModelInfo("thenlper/gte-small", architecture="BertModel", enable_test=False),
+ EmbedModelInfo(
+ "thenlper/gte-large-zh", architecture="BertModel", enable_test=False
+ ),
+ EmbedModelInfo("thenlper/gte-base-zh", architecture="BertModel", enable_test=False),
+ EmbedModelInfo(
+ "thenlper/gte-small-zh", architecture="BertModel", enable_test=False
+ ),
+ ########### NewModel
+ # These three architectures are almost the same, but not exactly the same.
+ # For example,
+ # - whether to use token_type_embeddings
+ # - whether to use context expansion
+ # So only test one (the most widely used) model
+ EmbedModelInfo(
+ "Alibaba-NLP/gte-multilingual-base",
+ architecture="GteNewModel",
+ mteb_score=0.775074696,
+ hf_overrides={"architectures": ["GteNewModel"]},
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ EmbedModelInfo(
+ "Alibaba-NLP/gte-base-en-v1.5",
+ architecture="GteNewModel",
+ hf_overrides={"architectures": ["GteNewModel"]},
+ enable_test=False,
+ ),
+ EmbedModelInfo(
+ "Alibaba-NLP/gte-large-en-v1.5",
+ architecture="GteNewModel",
+ hf_overrides={"architectures": ["GteNewModel"]},
+ enable_test=False,
+ ),
+ ########### Qwen2ForCausalLM
+ EmbedModelInfo(
+ "Alibaba-NLP/gte-Qwen2-1.5B-instruct",
+ mteb_score=0.758473459018872,
+ architecture="Qwen2ForCausalLM",
+ seq_pooling_type="LAST",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ # Skip: numerical regression with transformers v5.
+ enable_test=False,
+ ),
+ ########## ModernBertModel
+ EmbedModelInfo(
+ "Alibaba-NLP/gte-modernbert-base",
+ mteb_score=0.748193353,
+ architecture="ModernBertModel",
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ ########## Qwen3ForCausalLM
+ EmbedModelInfo(
+ "Qwen/Qwen3-Embedding-0.6B",
+ mteb_score=0.771163695,
+ architecture="Qwen3ForCausalLM",
+ seq_pooling_type="LAST",
+ attn_type="decoder",
+ is_prefix_caching_supported=True,
+ is_chunked_prefill_supported=True,
+ enable_test=True,
+ ),
+ EmbedModelInfo(
+ "Qwen/Qwen3-Embedding-4B",
+ architecture="Qwen3ForCausalLM",
+ enable_test=False,
+ ),
+]
+
+RERANK_MODELS = [
+ RerankModelInfo(
+ # classifier_pooling: mean
+ "Alibaba-NLP/gte-reranker-modernbert-base",
+ mteb_score=0.33386,
+ architecture="ModernBertForSequenceClassification",
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ RerankModelInfo(
+ "Alibaba-NLP/gte-multilingual-reranker-base",
+ mteb_score=0.33062,
+ architecture="GteNewForSequenceClassification",
+ hf_overrides={"architectures": ["GteNewForSequenceClassification"]},
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+]
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
+ mteb_test_embed_models(hf_runner, vllm_runner, model_info)
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_embed_models_correctness(
+ hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
+) -> None:
+ correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
+
+
+@pytest.mark.parametrize("model_info", RERANK_MODELS)
+def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
+ vllm_extra_kwargs = {}
+ if current_platform.is_rocm():
+ vllm_extra_kwargs["attention_backend"] = "TRITON_ATTN"
+ mteb_test_rerank_models(
+ vllm_runner, model_info, vllm_extra_kwargs=vllm_extra_kwargs
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_intfloat.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_intfloat.py
new file mode 100644
index 0000000000000000000000000000000000000000..345bce6b040ec822465313dd08476186672ae317
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_intfloat.py
@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+
+from tests.models.language.pooling.embed_utils import correctness_test_embed_models
+from tests.models.utils import EmbedModelInfo
+
+from .mteb_embed_utils import mteb_test_embed_models
+
+MODELS = [
+ ########## BertModel
+ EmbedModelInfo(
+ "intfloat/e5-small",
+ architecture="BertModel",
+ mteb_score=0.742285423,
+ seq_pooling_type="MEAN",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ EmbedModelInfo("intfloat/e5-base", architecture="BertModel", enable_test=False),
+ EmbedModelInfo("intfloat/e5-large", architecture="BertModel", enable_test=False),
+ EmbedModelInfo(
+ "intfloat/multilingual-e5-small", architecture="BertModel", enable_test=False
+ ),
+ ########## XLMRobertaModel
+ EmbedModelInfo(
+ "intfloat/multilingual-e5-base",
+ architecture="XLMRobertaModel",
+ mteb_score=0.779325955,
+ seq_pooling_type="MEAN",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ EmbedModelInfo(
+ "intfloat/multilingual-e5-large",
+ architecture="XLMRobertaModel",
+ enable_test=False,
+ ),
+ EmbedModelInfo(
+ "intfloat/multilingual-e5-large-instruct",
+ architecture="XLMRobertaModel",
+ enable_test=False,
+ ),
+]
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
+ mteb_test_embed_models(hf_runner, vllm_runner, model_info)
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_embed_models_correctness(
+ hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
+) -> None:
+ correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_jina.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_jina.py
new file mode 100644
index 0000000000000000000000000000000000000000..0f7329c1125dab15bae14e190b76bcb533e84380
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_jina.py
@@ -0,0 +1,154 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from functools import partial
+
+import pytest
+
+from tests.models.language.pooling.embed_utils import (
+ check_embeddings_close,
+ correctness_test_embed_models,
+ matryoshka_fy,
+)
+from tests.models.utils import (
+ EmbedModelInfo,
+ RerankModelInfo,
+)
+from vllm import PoolingParams
+
+from .mteb_embed_utils import mteb_test_embed_models
+from .mteb_score_utils import mteb_test_rerank_models
+
+EMBEDDING_MODELS = [
+ EmbedModelInfo(
+ "jinaai/jina-embeddings-v3",
+ mteb_score=0.824413164,
+ architecture="XLMRobertaModel",
+ is_matryoshka=True,
+ seq_pooling_type="MEAN",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ ),
+ EmbedModelInfo(
+ "jinaai/jina-embeddings-v5-text-small",
+ mteb_score=0.794535707854956,
+ architecture="JinaEmbeddingsV5Model",
+ seq_pooling_type="LAST",
+ attn_type="decoder",
+ is_prefix_caching_supported=True,
+ is_chunked_prefill_supported=True,
+ ),
+]
+
+RERANK_MODELS = [
+ RerankModelInfo(
+ "jinaai/jina-reranker-v2-base-multilingual",
+ mteb_score=0.33643,
+ architecture="XLMRobertaForSequenceClassification",
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ )
+]
+
+
+@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
+def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
+ task = "retrieval" if "v5" in model_info.name else "text-matching"
+ prompt_prefix: str | None = "Document: " if "v5" in model_info.name else None
+
+ def hf_model_callback(model):
+ model.encode = partial(model.encode, task=task)
+
+ mteb_test_embed_models(
+ hf_runner,
+ vllm_runner,
+ model_info,
+ hf_model_callback=hf_model_callback,
+ prompt_prefix=prompt_prefix,
+ )
+
+
+@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
+def test_embed_models_correctness(
+ hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
+) -> None:
+ task = "retrieval" if "v5" in model_info.name else "text-matching"
+
+ def hf_model_callback(model):
+ model.encode = partial(model.encode, task=task)
+
+ correctness_test_embed_models(
+ hf_runner,
+ vllm_runner,
+ model_info,
+ example_prompts,
+ hf_model_callback=hf_model_callback,
+ )
+
+
+@pytest.mark.parametrize("model_info", RERANK_MODELS)
+def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
+ mteb_test_rerank_models(vllm_runner, model_info)
+
+
+@pytest.mark.skip(
+ reason="jinaai/jina-embeddings-v3 custom XLMRobertaLoRA model on HF hub "
+ "is incompatible with transformers v5 (missing all_tied_weights_keys)"
+)
+@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
+@pytest.mark.parametrize("dtype", ["half"])
+@pytest.mark.parametrize("dimensions", [16, 32])
+def test_matryoshka(
+ hf_runner,
+ vllm_runner,
+ model_info,
+ dtype: str,
+ dimensions: int,
+ example_prompts,
+ monkeypatch,
+) -> None:
+ if not model_info.is_matryoshka:
+ pytest.skip("Model is not matryoshka")
+
+ # ST will strip the input texts, see test_embedding.py
+ example_prompts = [str(s).strip() for s in example_prompts]
+
+ task = "retrieval" if "v5" in model_info.name else "text-matching"
+
+ with hf_runner(
+ model_info.name,
+ dtype=dtype,
+ is_sentence_transformer=True,
+ ) as hf_model:
+ hf_outputs = hf_model.encode(example_prompts, task=task)
+ hf_outputs = matryoshka_fy(hf_outputs, dimensions)
+
+ with vllm_runner(
+ model_info.name, runner="pooling", dtype=dtype, max_model_len=None
+ ) as vllm_model:
+ assert vllm_model.llm.llm_engine.model_config.is_matryoshka
+
+ matryoshka_dimensions = (
+ vllm_model.llm.llm_engine.model_config.matryoshka_dimensions
+ )
+ assert matryoshka_dimensions is not None
+
+ if dimensions not in matryoshka_dimensions:
+ with pytest.raises(ValueError):
+ vllm_model.embed(
+ example_prompts, pooling_params=PoolingParams(dimensions=dimensions)
+ )
+ else:
+ vllm_outputs = vllm_model.embed(
+ example_prompts, pooling_params=PoolingParams(dimensions=dimensions)
+ )
+
+ check_embeddings_close(
+ embeddings_0_lst=hf_outputs,
+ embeddings_1_lst=vllm_outputs,
+ name_0="hf",
+ name_1="vllm",
+ tol=1e-2,
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_mxbai_rerank.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_mxbai_rerank.py
new file mode 100644
index 0000000000000000000000000000000000000000..cc069c19d8014378dd03b72b1de9f17a9a8f4812
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_mxbai_rerank.py
@@ -0,0 +1,109 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from typing import Any
+
+import mteb
+import numpy as np
+import pytest
+import torch
+from torch.utils.data import DataLoader
+
+from tests.conftest import HfRunner
+from tests.models.utils import RerankModelInfo
+
+from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models
+
+mxbai_rerank_hf_overrides = {
+ "architectures": ["Qwen2ForSequenceClassification"],
+ "classifier_from_token": ["0", "1"],
+ "method": "from_2_way_softmax",
+}
+
+RERANK_MODELS = [
+ RerankModelInfo(
+ "mixedbread-ai/mxbai-rerank-base-v2",
+ architecture="Qwen2ForSequenceClassification",
+ hf_overrides=mxbai_rerank_hf_overrides,
+ seq_pooling_type="LAST",
+ attn_type="decoder",
+ is_prefix_caching_supported=True,
+ is_chunked_prefill_supported=True,
+ chat_template_name="mxbai_rerank_v2.jinja",
+ mteb_score=0.33651,
+ enable_test=True,
+ ),
+ RerankModelInfo(
+ "mixedbread-ai/mxbai-rerank-large-v2",
+ architecture="Qwen2ForSequenceClassification",
+ hf_overrides=mxbai_rerank_hf_overrides,
+ chat_template_name="mxbai_rerank_v2.jinja",
+ enable_test=False,
+ ),
+]
+
+
+class MxbaiRerankerHfRunner(MtebCrossEncoderMixin, HfRunner):
+ def __init__(
+ self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
+ ) -> None:
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+
+ HfRunner.__init__(
+ self,
+ model_name=model_name,
+ auto_cls=AutoModelForCausalLM,
+ dtype=dtype,
+ **kwargs,
+ )
+
+ self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
+ self.yes_loc = self.tokenizer.convert_tokens_to_ids("1")
+ self.no_loc = self.tokenizer.convert_tokens_to_ids("0")
+
+ @torch.no_grad
+ def predict(
+ self,
+ inputs1: DataLoader[mteb.types.BatchedInput],
+ inputs2: DataLoader[mteb.types.BatchedInput],
+ *args,
+ **kwargs,
+ ) -> np.ndarray:
+ queries = [text for batch in inputs1 for text in batch["text"]]
+ corpus = [text for batch in inputs2 for text in batch["text"]]
+
+ tokenizer = self.tokenizer
+ prompts = []
+ for query, document in zip(queries, corpus):
+ conversation = [
+ {"role": "query", "content": query},
+ {"role": "document", "content": document},
+ ]
+
+ prompt = tokenizer.apply_chat_template(
+ conversation=conversation,
+ tools=None,
+ chat_template=self.chat_template,
+ tokenize=False,
+ )
+ prompts.append(prompt)
+
+ def compute_logits(inputs):
+ logits = self.model(**inputs).logits[:, -1, :]
+ yes_logits = logits[:, self.yes_loc]
+ no_logits = logits[:, self.no_loc]
+ logits = yes_logits - no_logits
+ scores = logits.float().sigmoid()
+ return scores
+
+ scores = []
+ for prompt in prompts:
+ inputs = tokenizer([prompt], return_tensors="pt")
+ inputs = self.wrap_device(inputs)
+ score = compute_logits(inputs)
+ scores.append(score[0].item())
+ return torch.Tensor(scores)
+
+
+@pytest.mark.parametrize("model_info", RERANK_MODELS)
+def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
+ mteb_test_rerank_models(vllm_runner, model_info, hf_runner=MxbaiRerankerHfRunner)
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_nemotron.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_nemotron.py
new file mode 100644
index 0000000000000000000000000000000000000000..f0ce1d16002bdf43bfd5d715948f829808eae8f6
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_nemotron.py
@@ -0,0 +1,50 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from tests.models.language.pooling_mteb_test.mteb_embed_utils import (
+ mteb_test_embed_models,
+)
+from tests.models.language.pooling_mteb_test.mteb_score_utils import (
+ mteb_test_rerank_models,
+)
+from tests.models.utils import (
+ EmbedModelInfo,
+ RerankModelInfo,
+)
+
+EMBEDDING_MODELS = [
+ EmbedModelInfo(
+ "nvidia/llama-nemotron-embed-1b-v2",
+ architecture="LlamaBidirectionalModel",
+ mteb_score=0.689164662128673,
+ seq_pooling_type="MEAN",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ )
+]
+
+RERANK_MODELS = [
+ RerankModelInfo(
+ "nvidia/llama-nemotron-rerank-1b-v2",
+ architecture="LlamaBidirectionalForSequenceClassification",
+ chat_template_name="nemotron-rerank.jinja",
+ mteb_score=0.33994,
+ seq_pooling_type="MEAN",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ ),
+]
+
+
+@pytest.mark.parametrize("model_info", EMBEDDING_MODELS)
+def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
+ mteb_test_embed_models(hf_runner, vllm_runner, model_info)
+
+
+@pytest.mark.parametrize("model_info", RERANK_MODELS)
+def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
+ mteb_test_rerank_models(vllm_runner, model_info)
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_nomic.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_nomic.py
new file mode 100644
index 0000000000000000000000000000000000000000..f8de6fe00a94444e1f49bdd95ce1674c54990d96
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_nomic.py
@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from tests.models.language.pooling.embed_utils import correctness_test_embed_models
+from tests.models.utils import EmbedModelInfo
+
+from .mteb_embed_utils import mteb_test_embed_models
+
+MODELS = [
+ EmbedModelInfo(
+ "nomic-ai/nomic-embed-text-v1",
+ architecture="NomicBertModel",
+ # Fixme:
+ # Update nomic-embed code to support the latest
+ # HF version and remove revision set.
+ revision="720244025c1a7e15661a174c63cce63c8218e52b",
+ mteb_score=0.737568559,
+ enable_test=True,
+ seq_pooling_type="MEAN",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ ),
+ EmbedModelInfo(
+ "nomic-ai/nomic-embed-text-v1.5",
+ architecture="NomicBertModel",
+ enable_test=False,
+ ),
+ EmbedModelInfo(
+ "nomic-ai/CodeRankEmbed", architecture="NomicBertModel", enable_test=False
+ ),
+ EmbedModelInfo(
+ "nomic-ai/nomic-embed-text-v2-moe",
+ architecture="NomicBertModel",
+ mteb_score=0.715488912,
+ enable_test=True,
+ seq_pooling_type="MEAN",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ ),
+]
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
+ mteb_test_embed_models(hf_runner, vllm_runner, model_info)
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_embed_models_correctness(
+ hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
+) -> None:
+ correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_qwen3_reranker.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_qwen3_reranker.py
new file mode 100644
index 0000000000000000000000000000000000000000..5e935449416d8bebf399a333930812954c60d3e3
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_qwen3_reranker.py
@@ -0,0 +1,130 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# ruff: noqa: E501
+from typing import Any
+
+import mteb
+import numpy as np
+import pytest
+import torch
+from torch.utils.data import DataLoader
+
+from tests.conftest import HfRunner
+from tests.models.utils import RerankModelInfo
+from tests.utils import multi_gpu_test
+
+from .mteb_score_utils import MtebCrossEncoderMixin, mteb_test_rerank_models
+
+qwen3_reranker_hf_overrides = {
+ "architectures": ["Qwen3ForSequenceClassification"],
+ "classifier_from_token": ["no", "yes"],
+ "is_original_qwen3_reranker": True,
+}
+
+RERANK_MODELS = [
+ RerankModelInfo(
+ "Qwen/Qwen3-Reranker-0.6B",
+ architecture="Qwen3ForSequenceClassification",
+ hf_overrides=qwen3_reranker_hf_overrides,
+ chat_template_name="qwen3_reranker.jinja",
+ seq_pooling_type="LAST",
+ attn_type="decoder",
+ is_prefix_caching_supported=True,
+ is_chunked_prefill_supported=True,
+ mteb_score=0.33459,
+ enable_test=True,
+ ),
+ RerankModelInfo(
+ "Qwen/Qwen3-Reranker-4B",
+ architecture="Qwen3ForSequenceClassification",
+ chat_template_name="qwen3_reranker.jinja",
+ hf_overrides=qwen3_reranker_hf_overrides,
+ enable_test=False,
+ ),
+]
+
+
+class Qwen3RerankerHfRunner(MtebCrossEncoderMixin, HfRunner):
+ def __init__(
+ self, model_name: str, dtype: str = "auto", *args: Any, **kwargs: Any
+ ) -> None:
+ from transformers import AutoModelForCausalLM, AutoTokenizer
+
+ HfRunner.__init__(
+ self,
+ model_name=model_name,
+ auto_cls=AutoModelForCausalLM,
+ dtype=dtype,
+ **kwargs,
+ )
+
+ self.tokenizer = AutoTokenizer.from_pretrained(model_name, padding_side="left")
+ self.token_false_id = self.tokenizer.convert_tokens_to_ids("no")
+ self.token_true_id = self.tokenizer.convert_tokens_to_ids("yes")
+ self.max_length = 40960
+
+ @torch.no_grad
+ def predict(
+ self,
+ inputs1: DataLoader[mteb.types.BatchedInput],
+ inputs2: DataLoader[mteb.types.BatchedInput],
+ *args,
+ **kwargs,
+ ) -> np.ndarray:
+ queries = [text for batch in inputs1 for text in batch["text"]]
+ corpus = [text for batch in inputs2 for text in batch["text"]]
+
+ tokenizer = self.tokenizer
+ prompts = []
+ for query, document in zip(queries, corpus):
+ conversation = [
+ {"role": "query", "content": query},
+ {"role": "document", "content": document},
+ ]
+
+ prompt = tokenizer.apply_chat_template(
+ conversation=conversation,
+ tools=None,
+ chat_template=self.chat_template,
+ tokenize=False,
+ )
+ prompts.append(prompt)
+
+ def compute_logits(inputs):
+ batch_scores = self.model(**inputs).logits[:, -1, :]
+ true_vector = batch_scores[:, self.token_true_id]
+ false_vector = batch_scores[:, self.token_false_id]
+ batch_scores = torch.stack([false_vector, true_vector], dim=1)
+ batch_scores = torch.nn.functional.log_softmax(batch_scores, dim=1)
+ scores = batch_scores[:, 1].exp()
+ return scores
+
+ scores = []
+ for prompt in prompts:
+ inputs = tokenizer([prompt], return_tensors="pt")
+ inputs = self.wrap_device(inputs)
+ score = compute_logits(inputs)
+ scores.append(score[0].item())
+ return torch.Tensor(scores)
+
+
+@pytest.mark.parametrize("model_info", RERANK_MODELS)
+def test_rerank_models_mteb(vllm_runner, model_info: RerankModelInfo) -> None:
+ mteb_test_rerank_models(vllm_runner, model_info, hf_runner=Qwen3RerankerHfRunner)
+
+
+@pytest.mark.parametrize("model_info", RERANK_MODELS)
+@multi_gpu_test(num_gpus=2)
+def test_rerank_models_mteb_tp(vllm_runner, model_info: RerankModelInfo) -> None:
+ assert model_info.architecture == "Qwen3ForSequenceClassification"
+
+ vllm_extra_kwargs: dict[str, Any] = {
+ "tensor_parallel_size": 2,
+ }
+
+ mteb_test_rerank_models(
+ vllm_runner,
+ model_info,
+ vllm_extra_kwargs=vllm_extra_kwargs,
+ hf_runner=Qwen3RerankerHfRunner,
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py
new file mode 100644
index 0000000000000000000000000000000000000000..178fedb2f4ec69d929bdfc7773fdb83256b39bec
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_snowflake_arctic_embed.py
@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from tests.models.language.pooling.embed_utils import correctness_test_embed_models
+from tests.models.utils import EmbedModelInfo
+
+from .mteb_embed_utils import mteb_test_embed_models
+
+MODELS = [
+ EmbedModelInfo(
+ "Snowflake/snowflake-arctic-embed-xs",
+ is_matryoshka=False,
+ architecture="BertModel",
+ mteb_score=0.714927797,
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ EmbedModelInfo(
+ "Snowflake/snowflake-arctic-embed-s",
+ is_matryoshka=False,
+ architecture="BertModel",
+ enable_test=False,
+ ),
+ EmbedModelInfo(
+ "Snowflake/snowflake-arctic-embed-m",
+ is_matryoshka=False,
+ architecture="BertModel",
+ enable_test=False,
+ ),
+ EmbedModelInfo(
+ "Snowflake/snowflake-arctic-embed-m-long",
+ is_matryoshka=False,
+ architecture="NomicBertModel",
+ mteb_score=0.681146831,
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ EmbedModelInfo(
+ "Snowflake/snowflake-arctic-embed-l",
+ is_matryoshka=False,
+ architecture="BertModel",
+ enable_test=False,
+ ),
+ EmbedModelInfo(
+ "Snowflake/snowflake-arctic-embed-m-v1.5",
+ is_matryoshka=True,
+ architecture="BertModel",
+ mteb_score=0.649088363,
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ EmbedModelInfo(
+ "Snowflake/snowflake-arctic-embed-l-v2.0",
+ is_matryoshka=True,
+ architecture="XLMRobertaModel",
+ mteb_score=0.712258299,
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ EmbedModelInfo(
+ "Snowflake/snowflake-arctic-embed-m-v2.0",
+ is_matryoshka=True,
+ architecture="GteModel",
+ mteb_score=0.706622444,
+ seq_pooling_type="CLS",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+]
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
+ mteb_test_embed_models(hf_runner, vllm_runner, model_info)
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_embed_models_correctness(
+ hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
+) -> None:
+ correctness_test_embed_models(hf_runner, vllm_runner, model_info, example_prompts)
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_st_projector.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_st_projector.py
new file mode 100644
index 0000000000000000000000000000000000000000..89ad583c39aad0a274e3b891ddc44b5fca94179f
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_st_projector.py
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import pytest
+
+from tests.models.utils import (
+ EmbedModelInfo,
+)
+
+from .mteb_embed_utils import mteb_test_embed_models
+
+# ST models with projector (Dense) layers
+ST_PROJECTOR_MODELS = [
+ EmbedModelInfo(
+ "TencentBAC/Conan-embedding-v1",
+ architecture="BertModel",
+ mteb_score=0.688611955,
+ seq_pooling_type="MEAN",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+ EmbedModelInfo(
+ "google/embeddinggemma-300m",
+ architecture="Gemma3TextModel",
+ mteb_score=0.7473819294684156,
+ seq_pooling_type="MEAN",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ enable_test=True,
+ ),
+]
+
+
+@pytest.mark.parametrize("model_info", ST_PROJECTOR_MODELS)
+def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
+ mteb_test_embed_models(hf_runner, vllm_runner, model_info)
diff --git a/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_voyage.py b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_voyage.py
new file mode 100644
index 0000000000000000000000000000000000000000..643d4b6c7db989243e8d397373ac4d92b6da4919
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/language/pooling_mteb_test/test_voyage.py
@@ -0,0 +1,56 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+import pytest
+
+from tests.models.language.pooling.embed_utils import correctness_test_embed_models
+from tests.models.utils import EmbedModelInfo
+
+from .mteb_embed_utils import mteb_test_embed_models
+
+MODELS = [
+ EmbedModelInfo(
+ "voyageai/voyage-4-nano",
+ architecture="VoyageQwen3BidirectionalEmbedModel",
+ enable_test=True,
+ seq_pooling_type="MEAN",
+ attn_type="encoder_only",
+ is_prefix_caching_supported=False,
+ is_chunked_prefill_supported=False,
+ hf_overrides={
+ "architectures": ["VoyageQwen3BidirectionalEmbedModel"],
+ "num_labels": 2048,
+ },
+ mteb_score=0.7054,
+ # === MTEB Results ===
+ # STS12: 0.6613
+ # STS13: 0.6906
+ # STS14: 0.6556
+ # STS15: 0.7843
+ # STS16: 0.7340
+ # STSBenchmark: 0.7063
+ # Average score: 0.7054
+ ),
+]
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_embed_models_mteb(hf_runner, vllm_runner, model_info: EmbedModelInfo) -> None:
+ # Encoder-only attention models need enforce_eager=True to avoid
+ # CUDA graph capture issues with piecewise compilation
+ mteb_test_embed_models(
+ hf_runner, vllm_runner, model_info, vllm_extra_kwargs={"enforce_eager": True}
+ )
+
+
+@pytest.mark.parametrize("model_info", MODELS)
+def test_embed_models_correctness(
+ hf_runner, vllm_runner, model_info: EmbedModelInfo, example_prompts
+) -> None:
+ correctness_test_embed_models(
+ hf_runner,
+ vllm_runner,
+ model_info,
+ example_prompts,
+ vllm_extra_kwargs={"enforce_eager": True},
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/multimodal/__init__.py b/asian_ai/performance/vLLM/tests/models/multimodal/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/models/multimodal/conftest.py b/asian_ai/performance/vLLM/tests/models/multimodal/conftest.py
new file mode 100644
index 0000000000000000000000000000000000000000..a5bd24a2457e095c8ae5713ac73dfc32ea45420e
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/multimodal/conftest.py
@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Pytest configuration for vLLM multimodal tests."""
+
+import os
+import warnings
+
+import torch
+
+from vllm.platforms import current_platform
+
+
+def pytest_configure(config):
+ """Early ROCm configuration that must happen before test collection."""
+ if not current_platform.is_rocm():
+ return
+
+ # Disable skinny GEMM on ROCm to avoid non-deterministic results
+ # from atomic reductions in wvSplitKrc kernel.
+ # See: https://github.com/vllm-project/vllm/pull/33493#issuecomment-3906083975
+ os.environ["VLLM_ROCM_USE_SKINNY_GEMM"] = "0"
+ warnings.warn(
+ "ROCm: Set VLLM_ROCM_USE_SKINNY_GEMM=0 to avoid non-deterministic "
+ "results from skinny GEMM atomic reductions",
+ UserWarning,
+ stacklevel=1,
+ )
+
+
+def pytest_collection_modifyitems(config, items):
+ """Configure ROCm-specific settings based on collected tests."""
+ if not current_platform.is_rocm():
+ return
+
+ skip_patterns = ["test_granite_speech.py"]
+ if any(pattern in str(arg) for arg in config.args for pattern in skip_patterns):
+ return
+
+ # Disable Flash/MemEfficient SDP on ROCm to avoid HF Transformers
+ # accuracy issues: https://github.com/vllm-project/vllm/issues/30167
+ # TODO: Remove once ROCm SDP accuracy issues are resolved on HuggingFace
+ torch.backends.cuda.enable_flash_sdp(False)
+ torch.backends.cuda.enable_mem_efficient_sdp(False)
+ torch.backends.cuda.enable_math_sdp(True)
+ warnings.warn(
+ "ROCm: Disabled flash_sdp and mem_efficient_sdp, enabled math_sdp "
+ "to avoid HuggingFace Transformers accuracy issues",
+ UserWarning,
+ stacklevel=1,
+ )
+
+
+def patch_hf_vision_attn_for_rocm(model):
+ """Force SDPA for HF vision encoders on ROCm.
+
+ HF's flash_attention_2 has accuracy issues on ROCm that bypass
+ torch.backends.cuda settings. This forces SDPA which then uses
+ math_sdp via the pytest_collection_modifyitems settings.
+ """
+ if not current_platform.is_rocm():
+ return
+
+ inner = getattr(model, "model", model)
+
+ if hasattr(inner, "vision_embedding"):
+ vit = inner.vision_embedding[0]
+ for layer in vit.encoder.layers:
+ if hasattr(layer, "self_attn"):
+ layer.self_attn.vision_config._attn_implementation = "sdpa"
diff --git a/asian_ai/performance/vLLM/tests/models/multimodal/generation/__init__.py b/asian_ai/performance/vLLM/tests/models/multimodal/generation/__init__.py
new file mode 100644
index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391
diff --git a/asian_ai/performance/vLLM/tests/models/multimodal/generation/test_audioflamingo3.py b/asian_ai/performance/vLLM/tests/models/multimodal/generation/test_audioflamingo3.py
new file mode 100644
index 0000000000000000000000000000000000000000..0a7ffc3fa99678e2d33e312f59f525c4ee618d67
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/multimodal/generation/test_audioflamingo3.py
@@ -0,0 +1,171 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+# Copyright 2025 The vLLM team.
+# Copyright 2025 NVIDIA CORPORATION and the HuggingFace Inc. team. All rights
+# reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import json
+import os
+
+import pytest
+
+from tests.models.registry import HF_EXAMPLE_MODELS
+from vllm import LLM, SamplingParams
+
+MODEL_NAME = "nvidia/audio-flamingo-3-hf"
+SINGLE_CONVERSATION = [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "What is surprising about the relationship between "
+ "the barking and the music?",
+ },
+ {
+ "type": "audio_url",
+ "audio_url": {
+ "url": "https://huggingface.co/datasets/nvidia/AudioSkills/"
+ "resolve/main/assets/"
+ "dogs_barking_in_sync_with_the_music.wav",
+ },
+ },
+ ],
+ }
+]
+BATCHED_CONVERSATIONS = [
+ SINGLE_CONVERSATION,
+ [
+ {
+ "role": "user",
+ "content": [
+ {
+ "type": "text",
+ "text": "Why is the philosopher's name mentioned in the "
+ "lyrics? (A) To express a sense of nostalgia "
+ "(B) To indicate that language cannot express clearly, "
+ "satirizing the inversion of black and white in the world "
+ "(C) To add depth and complexity to the lyrics "
+ "(D) To showcase the wisdom and influence of the "
+ "philosopher",
+ },
+ {
+ "type": "audio_url",
+ "audio_url": {
+ "url": "https://huggingface.co/datasets/nvidia/"
+ "AudioSkills/resolve/main/assets/"
+ "Ch6Ae9DT6Ko_00-04-03_00-04-31.wav",
+ },
+ },
+ ],
+ }
+ ],
+]
+
+
+def get_fixture_path(filename):
+ return os.path.join(
+ os.path.dirname(__file__), "../../fixtures/audioflamingo3", filename
+ )
+
+
+def assert_output_matches(output, expected_text, expected_token_ids):
+ generated = output.outputs[0]
+ assert generated.text.strip() == expected_text
+ actual_token_ids = list(generated.token_ids)
+ assert (
+ actual_token_ids == expected_token_ids
+ or actual_token_ids == expected_token_ids[:-1]
+ or actual_token_ids[:-1] == expected_token_ids
+ )
+
+
+@pytest.fixture(scope="module")
+def llm():
+ model_info = HF_EXAMPLE_MODELS.get_hf_info("AudioFlamingo3ForConditionalGeneration")
+ model_info.check_transformers_version(on_fail="skip")
+
+ try:
+ return LLM(
+ model=MODEL_NAME,
+ dtype="bfloat16",
+ enforce_eager=True,
+ limit_mm_per_prompt={"audio": 1},
+ )
+ except Exception as e:
+ pytest.skip(f"Failed to load model {MODEL_NAME}: {e}")
+
+
+def test_single_generation(llm):
+ fixture_path = get_fixture_path("expected_results_single.json")
+ if not os.path.exists(fixture_path):
+ pytest.skip(f"Fixture not found: {fixture_path}")
+
+ with open(fixture_path) as f:
+ expected = json.load(f)
+
+ sampling_params = SamplingParams(temperature=0.0, max_tokens=128)
+
+ outputs = llm.chat(
+ messages=SINGLE_CONVERSATION,
+ sampling_params=sampling_params,
+ )
+ assert_output_matches(
+ outputs[0],
+ expected["transcriptions"][0],
+ expected["token_ids"][0],
+ )
+
+
+def test_batched_generation(llm):
+ fixture_path = get_fixture_path("expected_results_batched.json")
+ if not os.path.exists(fixture_path):
+ pytest.skip(f"Fixture not found: {fixture_path}")
+
+ with open(fixture_path) as f:
+ expected = json.load(f)
+
+ sampling_params = SamplingParams(temperature=0.0, max_tokens=128)
+
+ outputs = llm.chat(
+ messages=BATCHED_CONVERSATIONS,
+ sampling_params=sampling_params,
+ )
+
+ for i, output in enumerate(outputs):
+ assert_output_matches(
+ output,
+ expected["transcriptions"][i],
+ expected["token_ids"][i],
+ )
+
+
+def test_single_and_batched_generation_match(llm):
+ sampling_params = SamplingParams(temperature=0.0, max_tokens=128)
+
+ single_output = llm.chat(
+ messages=SINGLE_CONVERSATION,
+ sampling_params=sampling_params,
+ )[0]
+ batched_output = llm.chat(
+ messages=BATCHED_CONVERSATIONS,
+ sampling_params=sampling_params,
+ )[0]
+
+ assert single_output.outputs[0].text == batched_output.outputs[0].text
+ assert list(single_output.outputs[0].token_ids) == list(
+ batched_output.outputs[0].token_ids
+ )
diff --git a/asian_ai/performance/vLLM/tests/models/multimodal/generation/test_common.py b/asian_ai/performance/vLLM/tests/models/multimodal/generation/test_common.py
new file mode 100644
index 0000000000000000000000000000000000000000..2cdd7d8f8048b967acf0a755823a8b91ffb83764
--- /dev/null
+++ b/asian_ai/performance/vLLM/tests/models/multimodal/generation/test_common.py
@@ -0,0 +1,1391 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+"""Common tests for testing .generate() functionality for single / multiple
+image, embedding, and video support for different VLMs in vLLM.
+"""
+
+import math
+from collections import defaultdict
+from pathlib import PosixPath
+
+import pytest
+from packaging.version import Version
+from transformers import (
+ AutoModel,
+ AutoModelForCausalLM,
+ AutoModelForImageTextToText,
+ AutoModelForTextToWaveform,
+)
+from transformers import __version__ as TRANSFORMERS_VERSION
+
+from vllm.platforms import current_platform
+from vllm.utils.func_utils import identity
+
+from ....conftest import (
+ IMAGE_ASSETS,
+ AudioTestAssets,
+ HfRunner,
+ ImageTestAssets,
+ VideoTestAssets,
+ VllmRunner,
+)
+from ....utils import create_new_process_for_each_test, large_gpu_mark, multi_gpu_marks
+from ...utils import check_outputs_equal
+from .vlm_utils import custom_inputs, model_utils, runners
+from .vlm_utils.case_filtering import get_parametrized_options
+from .vlm_utils.types import (
+ CustomTestOptions,
+ ExpandableVLMTestArgs,
+ VLMTestInfo,
+ VLMTestType,
+)
+
+COMMON_BROADCAST_SETTINGS = {
+ "test_type": VLMTestType.IMAGE,
+ "dtype": "half",
+ "max_tokens": 5,
+ "tensor_parallel_size": 2,
+ "hf_model_kwargs": {"device_map": "auto"},
+ "image_size_factors": [(0.25, 0.5, 1.0)],
+ "distributed_executor_backend": (
+ "ray",
+ "mp",
+ ),
+}
+
+### Test configuration for specific models
+# NOTE: The convention of the test settings below is to lead each test key
+# with the name of the model arch used in the test, using underscores in place
+# of hyphens; this makes it more convenient to filter tests for a specific kind
+# of model. For example....
+#
+# To run all test types for a specific key:
+# use the k flag to substring match with a leading square bracket; if the
+# model arch happens to be a substring of another one, you can add a
+# trailing hyphen. E.g.,
+# - pytest $TEST_FILE -k "[llava-"
+# prevents matching on "[llava_next-" & will match just the enabled cases
+# for llava, i.e., single image, image embedding, and custom input tests.
+#
+# To run a test for a Test Info for just one of multiple models:
+# use the k flag to substring match the model name, e.g.,
+# - pytest $TEST_FILE -k OpenGVLab/InternVL2-1B
+# prevents matching on nGVLab/InternVL2-2B.
+#
+# You can also combine substrings to match more granularly.
+# ex 1:
+# pytest $TEST_FILE -k "test_single_image and OpenGVLab/InternVL2-1B"
+# will run only test_single_image* for OpenGVLab/InternVL2-1B; this would
+# match both wrappers for single image tests, since it also matches
+# test_single_image_heavy (which forks if we have a distributed backend)
+# ex 2:
+# pytest $TEST_FILE -k "[llava- or [intern_vl-"
+# will run all of the tests for only llava & internvl.
+#
+# NOTE you can add --collect-only to any of the above commands to see
+# which cases would be selected and deselected by pytest. In general,
+# this is a good idea for checking your command first, since tests are slow.
+
+
+def _granite4_vision_vllm_to_hf_output(vllm_output, model):
+ """Post-processor for granite4_vision vLLM output.
+
+ Self-contained to avoid calling AutoConfig/AutoTokenizer without
+ trust_remote_code (needed while the model is not in upstream HF).
+ """
+ output_ids, output_str, out_logprobs = vllm_output
+ mm_token_id = 100352
+ hf_output_ids = [
+ token_id
+ for idx, token_id in enumerate(output_ids)
+ if token_id != mm_token_id or idx == 0 or output_ids[idx - 1] != mm_token_id
+ ]
+ hf_output_str = (
+ output_str[1:] if output_str and output_str[0] == " " else output_str
+ )
+ eos_token_id = 100257
+ if hf_output_ids and hf_output_ids[-1] == eos_token_id:
+ hf_output_str = hf_output_str + "<|end_of_text|>"
+ return hf_output_ids, hf_output_str, out_logprobs
+
+
+VLM_TEST_SETTINGS = {
+ #### Core tests to always run in the CI
+ "llava": VLMTestInfo(
+ models=["llava-hf/llava-1.5-7b-hf"],
+ test_type=(VLMTestType.EMBEDDING, VLMTestType.IMAGE, VLMTestType.CUSTOM_INPUTS),
+ prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
+ convert_assets_to_embeddings=model_utils.get_llava_embeddings,
+ max_model_len=4096,
+ auto_cls=AutoModelForImageTextToText,
+ vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
+ custom_test_opts=[
+ CustomTestOptions(
+ inputs=custom_inputs.multi_image_multi_aspect_ratio_inputs(
+ formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:"
+ ),
+ limit_mm_per_prompt={"image": 4},
+ )
+ ],
+ vllm_runner_kwargs={"enable_mm_embeds": True},
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
+ ),
+ "paligemma": VLMTestInfo(
+ models=["google/paligemma-3b-mix-224"],
+ test_type=VLMTestType.IMAGE,
+ prompt_formatter=identity,
+ img_idx_to_prompt=lambda idx: "",
+ # Paligemma uses its own sample prompts because the default one fails
+ single_image_prompts=IMAGE_ASSETS.prompts(
+ {
+ "stop_sign": "caption es",
+ "cherry_blossom": "What is in the picture?",
+ }
+ ),
+ auto_cls=AutoModelForImageTextToText,
+ vllm_output_post_proc=model_utils.paligemma_vllm_to_hf_output,
+ ),
+ "qwen2_5_vl": VLMTestInfo(
+ models=["Qwen/Qwen2.5-VL-3B-Instruct"],
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
+ prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
+ img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
+ video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>",
+ enforce_eager=False,
+ max_model_len=4096,
+ max_num_seqs=2,
+ auto_cls=AutoModelForImageTextToText,
+ vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
+ image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
+ ),
+ "qwen2_5_omni": VLMTestInfo(
+ models=["Qwen/Qwen2.5-Omni-3B"],
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE, VLMTestType.VIDEO),
+ prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
+ img_idx_to_prompt=lambda idx: "<|vision_bos|><|IMAGE|><|vision_eos|>",
+ video_idx_to_prompt=lambda idx: "<|vision_bos|><|VIDEO|><|vision_eos|>",
+ max_model_len=4096,
+ max_num_seqs=2,
+ num_logprobs=6 if current_platform.is_cpu() else 5,
+ auto_cls=AutoModelForTextToWaveform,
+ vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
+ patch_hf_runner=model_utils.qwen2_5_omni_patch_hf_runner,
+ image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
+ marks=[pytest.mark.core_model, pytest.mark.cpu_model],
+ ),
+ "qwen3_vl": VLMTestInfo(
+ models=["Qwen/Qwen3-VL-4B-Instruct"],
+ test_type=(
+ VLMTestType.IMAGE,
+ VLMTestType.MULTI_IMAGE,
+ VLMTestType.VIDEO,
+ ),
+ enforce_eager=False,
+ needs_video_metadata=True,
+ prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
+ img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
+ video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
+ max_model_len=4096,
+ max_num_seqs=2,
+ num_logprobs=20,
+ auto_cls=AutoModelForImageTextToText,
+ vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
+ patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner,
+ image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
+ marks=[
+ pytest.mark.core_model,
+ ],
+ vllm_runner_kwargs={"attention_backend": "TRITON_ATTN"}
+ if current_platform.is_rocm()
+ else {},
+ ),
+ "ultravox": VLMTestInfo(
+ models=["fixie-ai/ultravox-v0_5-llama-3_2-1b"],
+ test_type=VLMTestType.AUDIO,
+ prompt_formatter=lambda audio_prompt: f"<|begin_of_text|><|start_header_id|>user<|end_header_id|>\n\n{audio_prompt}<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", # noqa: E501
+ audio_idx_to_prompt=lambda idx: "<|audio|>",
+ max_model_len=4096,
+ max_num_seqs=2,
+ auto_cls=AutoModel,
+ hf_output_post_proc=model_utils.ultravox_trunc_hf_output,
+ marks=[
+ pytest.mark.core_model,
+ pytest.mark.cpu_model,
+ # TODO: Remove skip once model has been upstreamed to Transformers
+ pytest.mark.skip(
+ reason="Custom model code is not compatible with Transformers v5"
+ ),
+ ],
+ ),
+ #### Transformers fallback to test
+ ## To reduce test burden, we only test batching arbitrary image size
+ # Dynamic image length and number of patches
+ "llava-onevision-transformers": VLMTestInfo(
+ models=["llava-hf/llava-onevision-qwen2-0.5b-ov-hf"],
+ test_type=VLMTestType.IMAGE,
+ prompt_formatter=lambda vid_prompt: f"<|im_start|>user\n{vid_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
+ max_model_len=16384,
+ hf_model_kwargs=model_utils.llava_onevision_hf_model_kwargs(
+ "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
+ ),
+ auto_cls=AutoModelForImageTextToText,
+ vllm_output_post_proc=model_utils.llava_onevision_vllm_to_hf_output,
+ image_size_factors=[(0.25, 0.5, 1.0)],
+ vllm_runner_kwargs={
+ "model_impl": "transformers",
+ "default_torch_num_threads": 1,
+ },
+ marks=[pytest.mark.core_model],
+ ),
+ # Gemma3 has bidirectional mask on images
+ "gemma3-transformers": VLMTestInfo(
+ models=["google/gemma-3-4b-it"],
+ test_type=VLMTestType.IMAGE,
+ prompt_formatter=lambda vid_prompt: f"<'user\n{vid_prompt}\nmodel\n", # noqa: E501
+ max_model_len=4096,
+ auto_cls=AutoModelForImageTextToText,
+ vllm_output_post_proc=model_utils.gemma3_vllm_to_hf_output,
+ image_size_factors=[(0.25, 0.5, 1.0)],
+ vllm_runner_kwargs={
+ "model_impl": "transformers",
+ },
+ marks=[
+ pytest.mark.core_model,
+ *([large_gpu_mark(min_gb=80)] if current_platform.is_rocm() else []),
+ ],
+ ),
+ "idefics3-transformers": VLMTestInfo(
+ models=["HuggingFaceTB/SmolVLM-256M-Instruct"],
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
+ prompt_formatter=lambda img_prompt: f"<|begin_of_text|>User:{img_prompt}\nAssistant:", # noqa: E501
+ img_idx_to_prompt=lambda idx: "",
+ max_model_len=8192,
+ max_num_seqs=2,
+ auto_cls=AutoModelForImageTextToText,
+ hf_output_post_proc=model_utils.idefics3_trunc_hf_output,
+ image_size_factors=[(0.25, 0.5, 1.0)],
+ vllm_runner_kwargs={
+ "model_impl": "transformers",
+ },
+ marks=[pytest.mark.core_model],
+ ),
+ # Pixel values from processor are not 4D or 5D arrays
+ "qwen2_5_vl-transformers": VLMTestInfo(
+ models=["Qwen/Qwen2.5-VL-3B-Instruct"],
+ test_type=VLMTestType.IMAGE,
+ prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
+ img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>",
+ max_model_len=4096,
+ max_num_seqs=2,
+ auto_cls=AutoModelForImageTextToText,
+ vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
+ image_size_factors=[(0.25, 0.2, 0.15)],
+ vllm_runner_kwargs={
+ "model_impl": "transformers",
+ # TODO: [ROCm] Revert this once issue #30167 is resolved
+ **(
+ {
+ "mm_processor_kwargs": {
+ "min_pixels": 256 * 28 * 28,
+ "max_pixels": 1280 * 28 * 28,
+ },
+ }
+ if current_platform.is_rocm()
+ else {}
+ ),
+ },
+ marks=[large_gpu_mark(min_gb=80 if current_platform.is_rocm() else 32)],
+ ),
+ #### Extended model tests
+ "aria": VLMTestInfo(
+ models=["rhymes-ai/Aria"],
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
+ prompt_formatter=lambda img_prompt: f"<|im_start|>user\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n ", # noqa: E501
+ img_idx_to_prompt=lambda idx: "<|img|>\n",
+ max_model_len=4096,
+ max_num_seqs=2,
+ auto_cls=AutoModelForImageTextToText,
+ single_image_prompts=IMAGE_ASSETS.prompts(
+ {
+ "stop_sign": "Please describe the image shortly.",
+ "cherry_blossom": "Please infer the season with reason.",
+ }
+ ),
+ multi_image_prompt="Describe the two images shortly.",
+ stop_str=["<|im_end|>"],
+ image_size_factors=[(0.10, 0.15)],
+ max_tokens=64,
+ marks=[
+ pytest.mark.skip(
+ reason="Aria needs to update for latest transformers, "
+ "must have a vision_processor.py."
+ "An issue has been filed:"
+ "https://huggingface.co/rhymes-ai/Aria/discussions/23"
+ ),
+ large_gpu_mark(min_gb=64),
+ ],
+ ),
+ "blip2": VLMTestInfo(
+ models=["Salesforce/blip2-opt-2.7b"],
+ test_type=VLMTestType.IMAGE,
+ prompt_formatter=lambda img_prompt: f"Question: {img_prompt} Answer:",
+ img_idx_to_prompt=lambda idx: "",
+ auto_cls=AutoModelForImageTextToText,
+ vllm_output_post_proc=model_utils.blip2_vllm_to_hf_output,
+ # FIXME: https://github.com/huggingface/transformers/pull/38510
+ marks=[pytest.mark.skip("Model is broken")],
+ ),
+ "chameleon": VLMTestInfo(
+ models=["facebook/chameleon-7b"],
+ test_type=VLMTestType.IMAGE,
+ prompt_formatter=lambda img_prompt: f"USER: {img_prompt}\nASSISTANT:",
+ max_model_len=4096,
+ max_num_seqs=2,
+ auto_cls=AutoModelForImageTextToText,
+ # For chameleon, we only compare the sequences
+ vllm_output_post_proc=lambda vllm_output, model: vllm_output[:2],
+ hf_output_post_proc=lambda hf_output, model: hf_output[:2],
+ comparator=check_outputs_equal,
+ max_tokens=8,
+ dtype="bfloat16",
+ ),
+ "cosmos3": VLMTestInfo(
+ models=["nvidia/Cosmos3-Nano"],
+ test_type=(
+ VLMTestType.IMAGE,
+ VLMTestType.MULTI_IMAGE,
+ VLMTestType.VIDEO,
+ ),
+ enforce_eager=False,
+ needs_video_metadata=True,
+ prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>assistant\n", # noqa: E501
+ img_idx_to_prompt=lambda idx: "<|vision_start|><|image_pad|><|vision_end|>", # noqa: E501
+ video_idx_to_prompt=lambda idx: "<|vision_start|><|video_pad|><|vision_end|>", # noqa: E501
+ max_model_len=4096,
+ max_num_seqs=2,
+ num_logprobs=20,
+ auto_cls=AutoModelForImageTextToText,
+ vllm_output_post_proc=model_utils.qwen2_vllm_to_hf_output,
+ patch_hf_runner=model_utils.qwen3_vl_patch_hf_runner,
+ image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
+ ),
+ "deepseek_vl_v2": VLMTestInfo(
+ models=["Isotr0py/deepseek-vl2-tiny"], # model repo using dynamic module
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
+ prompt_formatter=lambda img_prompt: f"<|User|>: {img_prompt}\n\n<|Assistant|>: ", # noqa: E501
+ max_model_len=4096,
+ max_num_seqs=2,
+ single_image_prompts=IMAGE_ASSETS.prompts(
+ {
+ "stop_sign": "\nWhat's the content in the center of the image?",
+ "cherry_blossom": "\nPlease infer the season with reason in details.", # noqa: E501
+ }
+ ),
+ multi_image_prompt="image_1:\nimage_2:\nWhich image can we see the car and the tower?", # noqa: E501
+ patch_hf_runner=model_utils.deepseekvl2_patch_hf_runner,
+ hf_output_post_proc=model_utils.deepseekvl2_trunc_hf_output,
+ stop_str=["<|end▁of▁sentence|>", "<|begin▁of▁sentence|>"],
+ image_size_factors=[(1.0,), (1.0, 1.0, 1.0), (0.1, 0.5, 1.0)],
+ ),
+ "fuyu": VLMTestInfo(
+ models=["adept/fuyu-8b"],
+ test_type=VLMTestType.IMAGE,
+ prompt_formatter=lambda img_prompt: f"{img_prompt}\n",
+ img_idx_to_prompt=lambda idx: "",
+ max_model_len=2048,
+ max_num_seqs=2,
+ auto_cls=AutoModelForImageTextToText,
+ use_tokenizer_eos=True,
+ vllm_output_post_proc=model_utils.fuyu_vllm_to_hf_output,
+ num_logprobs=10,
+ image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
+ marks=[large_gpu_mark(min_gb=32)],
+ ),
+ "gemma3": VLMTestInfo(
+ models=["google/gemma-3-4b-it"],
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
+ prompt_formatter=lambda img_prompt: f"user\n{img_prompt}\nmodel\n", # noqa: E501
+ single_image_prompts=IMAGE_ASSETS.prompts(
+ {
+ "stop_sign": "What's the content in the center of the image?", # noqa: E501
+ "cherry_blossom": "What is the season?",
+ }
+ ),
+ multi_image_prompt="Describe the two images in detail.", # noqa: E501
+ max_model_len=4096,
+ max_num_seqs=2,
+ auto_cls=AutoModelForImageTextToText,
+ vllm_runner_kwargs={"mm_processor_kwargs": {"do_pan_and_scan": True}},
+ patch_hf_runner=model_utils.gemma3_patch_hf_runner,
+ ),
+ "gemma4": VLMTestInfo(
+ models=["google/gemma-4-E2B-it"],
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
+ prompt_formatter=lambda img_prompt: f"<|turn>user\n{img_prompt}\n<|turn>model\n", # noqa: E501
+ single_image_prompts=IMAGE_ASSETS.prompts(
+ {
+ "stop_sign": "<|image|>What's the content in the center of the image?", # noqa: E501
+ "cherry_blossom": "<|image|>What is the season?",
+ }
+ ),
+ multi_image_prompt="<|image|><|image|>Describe the two images in detail.", # noqa: E501
+ max_model_len=4096,
+ max_num_seqs=2,
+ auto_cls=AutoModelForImageTextToText,
+ vllm_runner_kwargs={"limit_mm_per_prompt": {"image": 4}},
+ ),
+ "granite_vision": VLMTestInfo(
+ models=["ibm-granite/granite-vision-3.3-2b"],
+ test_type=(VLMTestType.IMAGE),
+ prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}\n<|assistant|>\n",
+ max_model_len=8192,
+ auto_cls=AutoModelForImageTextToText,
+ vllm_output_post_proc=model_utils.llava_image_vllm_to_hf_output,
+ ),
+ "glm4v": VLMTestInfo(
+ models=["zai-org/glm-4v-9b"],
+ test_type=VLMTestType.IMAGE,
+ prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}<|assistant|>",
+ single_image_prompts=IMAGE_ASSETS.prompts(
+ {
+ "stop_sign": "<|begin_of_image|><|endoftext|><|end_of_image|>What's the content in the center of the image?", # noqa: E501
+ "cherry_blossom": "<|begin_of_image|><|endoftext|><|end_of_image|>What is the season?", # noqa: E501
+ }
+ ),
+ max_model_len=2048,
+ max_num_seqs=2,
+ get_stop_token_ids=lambda tok: [151329, 151336, 151338],
+ patch_hf_runner=model_utils.glm4v_patch_hf_runner,
+ # The image embeddings match with HF but the outputs of the language
+ # decoder are only consistent up to 2 decimal places.
+ # So, we need to reduce the number of tokens for the test to pass.
+ max_tokens=8,
+ num_logprobs=10,
+ auto_cls=AutoModelForCausalLM,
+ marks=[
+ pytest.mark.skip(
+ reason="The code for this model has a bug."
+ "Please see the issue here:"
+ "https://huggingface.co/zai-org/glm-4v-9b/discussions/46."
+ ),
+ large_gpu_mark(min_gb=32),
+ ],
+ ),
+ "glm4_1v": VLMTestInfo(
+ models=["zai-org/GLM-4.1V-9B-Thinking"],
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
+ prompt_formatter=lambda img_prompt: f"[gMASK]<|user|>\n{img_prompt}<|assistant|>\n", # noqa: E501
+ img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>",
+ video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>",
+ max_model_len=2048,
+ max_num_seqs=2,
+ get_stop_token_ids=lambda tok: [151329, 151336, 151338],
+ num_logprobs=10,
+ image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
+ auto_cls=AutoModelForImageTextToText,
+ marks=[large_gpu_mark(min_gb=32)],
+ ),
+ "glm4_1v-video": VLMTestInfo(
+ models=["zai-org/GLM-4.1V-9B-Thinking"],
+ # GLM4.1V require include video metadata for input
+ test_type=VLMTestType.CUSTOM_INPUTS,
+ prompt_formatter=lambda vid_prompt: f"[gMASK]<|user|>\n{vid_prompt}<|assistant|>\n", # noqa: E501
+ max_model_len=4096,
+ max_num_seqs=2,
+ auto_cls=AutoModelForImageTextToText,
+ patch_hf_runner=model_utils.glm4_1v_patch_hf_runner,
+ custom_test_opts=[
+ CustomTestOptions(
+ inputs=custom_inputs.video_with_metadata_glm4_1v(),
+ limit_mm_per_prompt={"video": 1},
+ )
+ ],
+ marks=[large_gpu_mark(min_gb=32)],
+ ),
+ "glm_ocr": VLMTestInfo(
+ models=["zai-org/GLM-OCR"],
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
+ prompt_formatter=lambda img_prompt: f"[gMASK]<|user|>\n{img_prompt}<|assistant|>\n", # noqa: E501
+ img_idx_to_prompt=lambda idx: "<|begin_of_image|><|image|><|end_of_image|>",
+ video_idx_to_prompt=lambda idx: "<|begin_of_video|><|video|><|end_of_video|>",
+ max_model_len=2048,
+ max_num_seqs=2,
+ get_stop_token_ids=lambda tok: [151329, 151336, 151338],
+ num_logprobs=10,
+ image_size_factors=[(0.25,), (0.25, 0.25, 0.25), (0.25, 0.2, 0.15)],
+ auto_cls=AutoModelForImageTextToText,
+ marks=[
+ pytest.mark.skip(
+ reason="This test fails on both AMD and NV"
+ "hardware. please see the issue:"
+ "https://github.com/vllm-project/vllm/issues/42016"
+ ),
+ large_gpu_mark(min_gb=32),
+ ],
+ ),
+ "granite4_vision": VLMTestInfo(
+ models=["ibm-granite/granite-vision-4.1-4b"],
+ test_type=(VLMTestType.IMAGE),
+ prompt_formatter=lambda img_prompt: f"<|user|>\n{img_prompt}\n<|assistant|>\n",
+ max_model_len=8192,
+ auto_cls=AutoModelForImageTextToText,
+ vllm_output_post_proc=_granite4_vision_vllm_to_hf_output,
+ image_size_factors=[(1.0,)],
+ vllm_runner_kwargs={
+ "enable_lora": True,
+ "max_lora_rank": 256,
+ "default_mm_loras": {"image": "ibm-granite/granite-vision-4.1-4b"},
+ },
+ ),
+ "h2ovl": VLMTestInfo(
+ models=[
+ "h2oai/h2ovl-mississippi-800m",
+ "h2oai/h2ovl-mississippi-2b",
+ ],
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
+ prompt_formatter=lambda img_prompt: f"<|prompt|>{img_prompt}<|end|><|answer|>",
+ single_image_prompts=IMAGE_ASSETS.prompts(
+ {
+ "stop_sign": "\nWhat's the content in the center of the image?",
+ "cherry_blossom": "\nWhat is the season?",
+ }
+ ),
+ multi_image_prompt="Image-1: \nImage-2: \nDescribe the two images in short.", # noqa: E501
+ max_model_len=8192,
+ use_tokenizer_eos=True,
+ num_logprobs=10,
+ patch_hf_runner=model_utils.h2ovl_patch_hf_runner,
+ ),
+ "idefics3": VLMTestInfo(
+ models=["HuggingFaceTB/SmolVLM-256M-Instruct"],
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
+ prompt_formatter=lambda img_prompt: f"<|begin_of_text|>User:{img_prompt}\nAssistant:", # noqa: E501
+ img_idx_to_prompt=lambda idx: "",
+ max_model_len=8192,
+ max_num_seqs=2,
+ auto_cls=AutoModelForImageTextToText,
+ hf_output_post_proc=model_utils.idefics3_trunc_hf_output,
+ ),
+ "intern_vl": VLMTestInfo(
+ models=[
+ "OpenGVLab/InternVL2-1B",
+ "OpenGVLab/InternVL2-2B",
+ ],
+ test_type=(VLMTestType.IMAGE, VLMTestType.MULTI_IMAGE),
+ prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
+ single_image_prompts=IMAGE_ASSETS.prompts(
+ {
+ "stop_sign": "\nWhat's the content in the center of the image?",
+ "cherry_blossom": "\nWhat is the season?",
+ }
+ ),
+ multi_image_prompt="Image-1: \nImage-2: \nDescribe the two images in short.", # noqa: E501
+ max_model_len=4096,
+ use_tokenizer_eos=True,
+ patch_hf_runner=model_utils.internvl_patch_hf_runner,
+ # TODO: Remove skip once model has been upstreamed to Transformers
+ marks=[
+ pytest.mark.skip(
+ reason="Custom model code tries to access data from meta-tensor"
+ )
+ ],
+ ),
+ "intern_vl-video": VLMTestInfo(
+ models=[
+ "OpenGVLab/InternVL3-1B",
+ ],
+ test_type=VLMTestType.VIDEO,
+ prompt_formatter=lambda img_prompt: f"<|im_start|>User\n{img_prompt}<|im_end|>\n<|im_start|>Assistant\n", # noqa: E501
+ video_idx_to_prompt=lambda idx: "