init v0.23.0

Signed-off-by: Sun Ruoxi <sunruoxi@4paradigm.com>

tests/e2e/pull_request/one_card/lora/test_ilama_lora.py

@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: Apache-2.0
+import vllm
+from vllm.lora.request import LoRARequest
+
+from tests.e2e.conftest import VllmRunner
+
+MODEL_PATH = "vllm-ascend/ilama-3.2-1B"
+
+PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.\n"\n##Instruction:\nconcert_singer contains tables such as stadium, singer, concert, singer_in_concert. Table stadium has columns such as Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average. Stadium_ID is the primary key.\nTable singer has columns such as Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male. Singer_ID is the primary key.\nTable concert has columns such as concert_ID, concert_Name, Theme, Stadium_ID, Year. concert_ID is the primary key.\nTable singer_in_concert has columns such as concert_ID, Singer_ID. concert_ID is the primary key.\nThe Stadium_ID of concert is the foreign key of Stadium_ID of stadium.\nThe Singer_ID of singer_in_concert is the foreign key of Singer_ID of singer.\nThe concert_ID of singer_in_concert is the foreign key of concert_ID of concert.\n\n###Input:\n{query}\n\n###Response:"""  # noqa: E501
+
+EXPECTED_LORA_OUTPUT = [
+    "SELECT count(*) FROM singer",
+    "SELECT avg(age) ,  min(age) ,  max(age) FROM singer WHERE country  =  'France'",  # noqa: E501
+    "SELECT DISTINCT Country FROM singer WHERE Age  >  20",
+]
+
+
+def do_sample(llm: vllm.LLM, lora_path: str, lora_id: int) -> list[str]:
+    prompts = [
+        PROMPT_TEMPLATE.format(query="How many singers do we have?"),
+        PROMPT_TEMPLATE.format(
+            query="What is the average, minimum, and maximum age of all singers from France?"  # noqa: E501
+        ),
+        PROMPT_TEMPLATE.format(
+            query="What are all distinct countries where singers above age 20 are from?"  # noqa: E501
+        ),
+    ]
+    sampling_params = vllm.SamplingParams(temperature=0, max_tokens=32)
+    outputs = llm.generate(
+        prompts, sampling_params, lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None
+    )
+    # Print the outputs.
+    generated_texts: list[str] = []
+    for output in outputs:
+        prompt = output.prompt
+        generated_text = output.outputs[0].text.strip()
+        generated_texts.append(generated_text)
+        print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+    return generated_texts
+
+
+def test_ilama_lora(ilama_lora_files):
+    with VllmRunner(
+        MODEL_PATH,
+        enable_lora=True,
+        dtype="half",
+        max_loras=4,
+        max_model_len=1024,
+        cudagraph_capture_sizes=[1, 2, 4, 8],
+        max_num_seqs=16,
+        enforce_eager=True,
+    ) as vllm_model:
+        output1 = do_sample(vllm_model.model, ilama_lora_files, lora_id=1)
+        for i in range(len(EXPECTED_LORA_OUTPUT)):
+            assert output1[i] == EXPECTED_LORA_OUTPUT[i]
+
+        output2 = do_sample(vllm_model.model, ilama_lora_files, lora_id=2)
+        for i in range(len(EXPECTED_LORA_OUTPUT)):
+            assert output2[i] == EXPECTED_LORA_OUTPUT[i]

tests/e2e/pull_request/one_card/lora/test_llama32_lora.py

@@ -0,0 +1,140 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+
+from unittest.mock import patch
+
+import vllm
+import vllm.config
+from vllm.lora.request import LoRARequest
+
+from tests.e2e.conftest import VllmRunner
+from vllm_ascend.utils import enable_custom_op
+
+enable_custom_op()
+
+PROMPT_TEMPLATE = """<|eot_id|><|start_header_id|>user<|end_header_id|>
+I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me.Below is an instruction that describes a task, Write a response that appropriately completes the request.
+"
+##Instruction:
+candidate_poll contains tables such as candidate, people. Table candidate has columns such as Candidate_ID, People_ID, Poll_Source, Date, Support_rate, Consider_rate, Oppose_rate, Unsure_rate. Candidate_ID is the primary key.
+Table people has columns such as People_ID, Sex, Name, Date_of_Birth, Height, Weight. People_ID is the primary key.
+The People_ID of candidate is the foreign key of People_ID of people.
+###Input:
+{context}
+###Response:<|eot_id|><|start_header_id|>assistant<|end_header_id|>
+"""  # noqa: E501
+
+EXPECTED_LORA_OUTPUT = [
+    "SELECT count(*) FROM candidate",
+    "SELECT count(*) FROM candidate",
+    "SELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1",  # noqa: E501
+    "SELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1",  # noqa: E501
+]
+
+EXPECTED_BASE_MODEL_OUTPUT = [
+    "SELECT COUNT(*) FROM candidate",
+    "`SELECT COUNT(*) FROM candidate;`",
+    "SELECT Poll_Source FROM candidate GROUP BY Poll_Source ORDER BY COUNT(*) DESC LIMIT 1;",
+    "SELECT * FROM candidate ORDER BY Candidate_ID DESC LIMIT 1",
+]
+
+# For hk region, we need to use the model from hf to avoid the network issue
+MODEL_PATH = "meta-llama/Llama-3.2-3B-Instruct"
+
+
+def do_sample(
+    llm: vllm.LLM,
+    lora_path: str,
+    lora_id: int,
+    tensorizer_config_dict: dict | None = None,
+) -> list[str]:
+    prompts = [
+        PROMPT_TEMPLATE.format(context="How many candidates are there?"),
+        PROMPT_TEMPLATE.format(context="Count the number of candidates."),
+        PROMPT_TEMPLATE.format(
+            context="Which poll resource provided the most number of candidate information?"  # noqa: E501
+        ),
+        PROMPT_TEMPLATE.format(context="Return the poll resource associated with the most candidates."),
+    ]
+
+    sampling_params = vllm.SamplingParams(temperature=0, max_tokens=64, stop=["<|im_end|>"])
+    if tensorizer_config_dict is not None:
+        outputs = llm.generate(
+            prompts,
+            sampling_params,
+            lora_request=LoRARequest(
+                str(lora_id),
+                lora_id,
+                lora_path,
+                tensorizer_config_dict=tensorizer_config_dict,
+            )
+            if lora_id
+            else None,
+        )
+    else:
+        outputs = llm.generate(
+            prompts,
+            sampling_params,
+            lora_request=LoRARequest(str(lora_id), lora_id, lora_path) if lora_id else None,
+        )
+
+    generated_texts: list[str] = []
+    for output in outputs:
+        prompt = output.prompt
+        generated_text = output.outputs[0].text
+        generated_texts.append(generated_text)
+        print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+    return generated_texts
+
+
+def generate_and_test(llm, llama32_lora_files, tensorizer_config_dict: dict | None = None):
+    print("lora adapter created")
+    print("lora 1")
+    assert (
+        do_sample(
+            llm,
+            llama32_lora_files,
+            tensorizer_config_dict=tensorizer_config_dict,
+            lora_id=1,
+        )
+        == EXPECTED_LORA_OUTPUT
+    )
+
+    print("lora 2")
+    assert (
+        do_sample(
+            llm,
+            llama32_lora_files,
+            tensorizer_config_dict=tensorizer_config_dict,
+            lora_id=2,
+        )
+        == EXPECTED_LORA_OUTPUT
+    )
+
+    print("base model")
+    assert (
+        do_sample(
+            llm,
+            llama32_lora_files,
+            tensorizer_config_dict=tensorizer_config_dict,
+            lora_id=0,
+        )
+        == EXPECTED_BASE_MODEL_OUTPUT
+    )
+
+    print("removing lora")
+
+
+@patch.dict("os.environ", {"VLLM_USE_MODELSCOPE": "False"})
+def test_llama_lora(llama32_lora_files):
+    vllm_model = VllmRunner(
+        MODEL_PATH,
+        enable_lora=True,
+        # also test odd max_num_seqs
+        max_num_seqs=7,
+        max_model_len=1024,
+        max_loras=4,
+        compilation_config={"cudagraph_mode": "PIECEWISE"},
+    )
+    llm = vllm_model.model
+    generate_and_test(llm, llama32_lora_files)

tests/e2e/pull_request/one_card/lora/test_lora_with_spec_decode.py

@@ -0,0 +1,116 @@
+"""
+This script contains:
+1. test lora with speculative decoding for batch inference
+"""
+
+import random
+
+import numpy as np
+import pytest
+import torch
+from vllm import LLM, SamplingParams
+from vllm.lora.request import LoRARequest
+
+LORA_TEST_PROMPT_MAP: dict[str, str] = {}
+
+LORA_TEST_PROMPT_MAP["vllm-ascend/qwen-linear-algebra-coder"] = """
+### INSTRUCTION:
+You are an AI assistant that generates Python code to solve linear
+algebra problems.
+
+### PROBLEM:
+Find the eigenvalues and eigenvectors of the following 3x3 matrix:
+[[3, 2, 0],
+ [2, 3, 0],
+ [0, 0, 2]]
+
+### OUTPUT FORMAT (STRICT):
+Numbers should be represented as integers only.
+
+### PYTHON SOLUTION:
+"""
+
+SEED = 42
+
+
+@pytest.mark.parametrize(
+    "model_setup",
+    [
+        (
+            "eagle3",
+            "Qwen/Qwen3-1.7B",
+            "vllm-ascend/Qwen3-1.7B_eagle3",
+            "vllm-ascend/qwen-linear-algebra-coder",
+            1,
+        )
+    ],
+)
+def test_batch_inference_correctness(
+    model_setup: tuple[str, str, str, str, int],
+):
+    """
+    Compare the outputs of a LLM with only Lora and a LLM with both SD and Lora.
+    Should be the same and no failure when doing batch inference.
+    model_setup: (method, model_name, spec_model_name, lora_path, tp_size)
+    """
+    # Disable randomness
+    torch.manual_seed(SEED)
+    np.random.seed(SEED)
+    random.seed(SEED)
+    torch.use_deterministic_algorithms(True)
+
+    method, model_name, spec_model_name, lora_path, tp_size = model_setup
+    prompts = [LORA_TEST_PROMPT_MAP[lora_path]] * 100
+    lora_request = LoRARequest("adapter", 1, lora_path)
+    sampling_params = SamplingParams(temperature=0.0, top_p=1.0, top_k=-1, seed=SEED, max_tokens=128)
+
+    # without speculative decoding
+    ref_llm = LLM(
+        model=model_name,
+        trust_remote_code=True,
+        tensor_parallel_size=tp_size,
+        max_model_len=2048,
+        max_num_seqs=4,
+        enable_lora=True,
+        max_loras=1,
+        max_cpu_loras=1,
+        max_lora_rank=16,
+    )
+    ref_outputs = ref_llm.generate(prompts, sampling_params, lora_request=lora_request)
+    del ref_llm
+
+    # speculative decoding
+    lora_spec_llm = LLM(
+        model=model_name,
+        trust_remote_code=True,
+        tensor_parallel_size=tp_size,
+        speculative_config={
+            "method": method,
+            "model": spec_model_name,
+            "num_speculative_tokens": 3,
+            "max_model_len": 2048,
+        },
+        max_model_len=2048,
+        max_num_seqs=4,
+        enable_lora=True,
+        max_loras=1,
+        max_cpu_loras=1,
+        max_lora_rank=16,
+    )
+    lora_spec_outputs = lora_spec_llm.generate(prompts, sampling_params, lora_request=lora_request)
+    del lora_spec_llm
+
+    matches = 0
+    misses = 0
+    for ref_output, spec_output in zip(ref_outputs, lora_spec_outputs):
+        if ref_output.outputs[0].text == spec_output.outputs[0].text:
+            matches += 1
+        else:
+            misses += 1
+            print(f"ref_output: {ref_output.outputs[0].text}")
+            print(f"spec_output: {spec_output.outputs[0].text}")
+
+    # Heuristic: expect at least 90% of the prompts to match exactly
+    # Upon failure, inspect the outputs to check for inaccuracy.
+    print(f"match ratio: {matches}/{len(ref_outputs)}")
+    assert matches > int(0.90 * len(ref_outputs))

tests/e2e/pull_request/one_card/lora/test_olmoe_lora.py

@@ -0,0 +1,106 @@
+from collections.abc import Sequence
+
+import vllm
+from vllm.lora.request import LoRARequest
+
+MODEL_PATH = "allenai/OLMoE-1B-7B-0125-Instruct"
+
+PROMPT_TEMPLATE = """I want you to act as a SQL terminal in front of an example database, you need only to return the sql command to me. Do not return any additional explanation. Below is an instruction that describes a task, Write a response that appropriately completes the request.
+"
+##Instruction:
+candidate_poll contains tables such as candidate, people. Table candidate has columns such as Candidate_ID, People_ID, Poll_Source, Date, Support_rate, Consider_rate, Oppose_rate, Unsure_rate. Candidate_ID is the primary key.
+Table people has columns such as People_ID, Sex, Name, Date_of_Birth, Height, Weight. People_ID is the primary key.
+The People_ID of candidate is the foreign key of People_ID of people.
+
+
+###Input:
+{context}
+
+###Response:"""  # noqa: E501
+
+EXPECTED_LORA_OUTPUT = [
+    "SELECT count(*) FROM candidate",
+    "SELECT count(*) FROM candidate",
+    "SELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1",  # noqa: E501
+    "SELECT poll_source FROM candidate GROUP BY poll_source ORDER BY count(*) DESC LIMIT 1",  # noqa: E501
+]
+
+EXPECTED_BASE_MODEL_OUTPUT = [
+    "SELECT COUNT(Candidate_ID) FROM candidate",
+    "SELECT COUNT(Candidate_ID) FROM candidate",
+    "SELECT Candidate_ID, COUNT(*) as Total_Candidates\nFROM candidate\nINNER JOIN people ON candidate.People_ID = people.People_ID",  # noqa: E501
+    # There are multiple acceptable responses
+    (
+        "SELECT Candidate_ID, Poll_Source FROM candidate WHERE People_ID IN (SELECT People_ID FROM people) ORDER BY COUNT(*) DESC LIMIT 1",  # noqa: E501
+        "SELECT Candidate_ID, Poll_Source FROM candidate WHERE COUNT(People_ID) = (SELECT COUNT(People_ID) FROM people) ORDER BY Candidate_ID DESC LIMIT 1",  # noqa: E501
+    ),
+]
+
+
+def _output_matches(generated: str, accepted: str | Sequence[str]) -> bool:
+    if isinstance(accepted, str):
+        accepted = (accepted,)
+    return any(generated.startswith(s) for s in accepted)
+
+
+def generate_and_test(
+    llm: vllm.LLM,
+    lora_path: str,
+    lora_id: list[int | None] | int | None,
+    compare_lower: bool = False,
+) -> None:
+    prompts = [
+        PROMPT_TEMPLATE.format(context="How many candidates are there?"),
+        PROMPT_TEMPLATE.format(context="Count the number of candidates."),
+        PROMPT_TEMPLATE.format(
+            context="Which poll resource provided the most number of candidate information?"  # noqa: E501
+        ),
+        PROMPT_TEMPLATE.format(context="Return the poll resource associated with the most candidates."),
+    ]
+
+    lora_request = None
+    if isinstance(lora_id, int):
+        lora_request = LoRARequest(str(lora_id), lora_id, lora_path)
+    elif isinstance(lora_id, list):
+        lora_request = [LoRARequest(str(i), i, lora_path) if i is not None else None for i in lora_id]
+
+    sampling_params = vllm.SamplingParams(temperature=0, max_tokens=64)
+    outputs = llm.generate(prompts, sampling_params, lora_request=lora_request)
+    # Print the outputs.
+    generated_texts: list[str] = []
+    for output in outputs:
+        prompt = output.prompt
+        generated_text = output.outputs[0].text.strip()
+        generated_texts.append(generated_text)
+        print(f"Prompt: {prompt!r}, Generated text: {generated_text!r}")
+
+    for i in range(len(EXPECTED_LORA_OUTPUT)):
+        req_lora_id = lora_id[i] if isinstance(lora_id, list) else lora_id
+        generated_text = generated_texts[i]
+        expected_output = EXPECTED_LORA_OUTPUT[i] if req_lora_id is not None else EXPECTED_BASE_MODEL_OUTPUT[i]
+
+        if compare_lower:
+            generated_text = generated_text.lower()
+            if isinstance(expected_output, str):
+                expected_output = (expected_output.lower(),)
+            else:
+                expected_output = tuple(s.lower() for s in expected_output)
+        assert _output_matches(generated_text, expected_output), (
+            f"Output {i}: {generated_text!r} does not match any of {expected_output!r}"
+        )
+
+
+def test_olmoe_lora(olmoe_lora_files):
+    # We enable enforce_eager=True here to reduce VRAM usage for lora-test CI,
+    # Otherwise, the lora-test will fail due to CUDA OOM.
+    llm = vllm.LLM(
+        MODEL_PATH,
+        max_model_len=1024,
+        enable_lora=True,
+        max_loras=4,
+        enforce_eager=False,
+        trust_remote_code=True,
+        enable_chunked_prefill=True,
+    )
+
+    generate_and_test(llm, olmoe_lora_files, lora_id=1)

tests/e2e/pull_request/one_card/lora/test_qwen35_densemodel_lora.py

@@ -0,0 +1,90 @@
+import vllm
+from transformers import AutoTokenizer
+from vllm.lora.request import LoRARequest
+
+MODEL_PATH = "Qwen/Qwen3.5-4B"
+TEXT_LORA_ID = 1
+
+# text-only task
+TEXT_PROMPT_TEMPLATE = """Write a SQL query for the given database.\nSchema:\nTables:\n  - stadium(Stadium_ID, Location, Name, Capacity, Highest, Lowest, Average)\n  - singer(Singer_ID, Name, Country, Song_Name, Song_release_year, Age, Is_male)\n  - concert(concert_ID, concert_Name, Theme, Stadium_ID, Year)\n  - singer_in_concert(concert_ID, Singer_ID)\n\nQuestion:\n{query}"""  # noqa: E501
+
+TEXT_EXPECTED_LORA_OUTPUT = [
+    "SELECT count(*) FROM singer",
+    "SELECT avg(age) ,  min(age) ,  max(age) FROM singer WHERE country  =  'France'",
+    "SELECT name FROM stadium WHERE stadium_id NOT IN (SELECT stadium_id FROM concert)",
+]
+
+
+TOKENIZER = AutoTokenizer.from_pretrained(MODEL_PATH, trust_remote_code=True)
+
+
+def _assert_exact_outputs(generated_texts: list[str], expected_outputs: list[str]) -> None:
+    assert generated_texts == expected_outputs
+
+
+def _run_text_lora_sample(
+    llm: vllm.LLM,
+    lora_path: str,
+    lora_id: int,
+) -> list[str]:
+    prompts = [
+        TEXT_PROMPT_TEMPLATE.format(query="How many singers do we have?"),
+        TEXT_PROMPT_TEMPLATE.format(
+            query=("What is the average, minimum, and maximum age of all singers from France?")
+        ),
+        TEXT_PROMPT_TEMPLATE.format(query="What are the names of the stadiums without any concerts?"),
+    ]
+    input_templates = []
+    for prompt_text in prompts:
+        messages = [{"role": "user", "content": prompt_text}]
+        prompt = TOKENIZER.apply_chat_template(
+            messages,
+            tokenize=False,
+            add_generation_prompt=True,
+            enable_thinking=False,  # disable thinking
+        )
+        input_templates.append(prompt)
+
+    outputs = llm.generate(
+        input_templates,
+        vllm.SamplingParams(temperature=0.01, max_tokens=512),
+        lora_request=LoRARequest(str(lora_id), lora_id, lora_path),
+    )
+
+    generated_texts: list[str] = []
+    for output in outputs:
+        generated_text = output.outputs[0].text.strip()
+        generated_texts.append(generated_text)
+        print(f"Prompt: {output.prompt!r}, Generated text: {generated_text!r}")
+    return generated_texts
+
+
+def _assert_qwen35_text_lora(
+    llm: vllm.LLM,
+    qwen35_text_lora_files: str,
+) -> None:
+    generated_texts = _run_text_lora_sample(
+        llm,
+        qwen35_text_lora_files,
+        TEXT_LORA_ID,
+    )
+
+    _assert_exact_outputs(generated_texts, TEXT_EXPECTED_LORA_OUTPUT)
+
+
+def test_qwen35_text_lora(qwen35_text_lora_files):
+    llm = vllm.LLM(
+        model=MODEL_PATH,
+        max_model_len=4096,
+        enable_lora=True,
+        max_loras=2,
+        max_num_seqs=4,
+        max_lora_rank=8,
+        enforce_eager=True,
+        trust_remote_code=True,
+    )
+
+    _assert_qwen35_text_lora(
+        llm,
+        qwen35_text_lora_files,
+    )

tests/e2e/pull_request/one_card/lora/test_qwen3_multi_loras.py

@@ -0,0 +1,150 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+from unittest.mock import patch
+
+from vllm import SamplingParams
+from vllm.lora.request import LoRARequest
+
+from tests.e2e.conftest import VllmRunner
+from vllm_ascend.utils import enable_custom_op
+
+enable_custom_op()
+
+MODEL_PATH = "Qwen/Qwen3-0.6B"
+LORA_NAME_PATH_MAP = {
+    "Alice": "charent/self_cognition_Alice",
+    "Bob": "charent/self_cognition_Bob",
+    "Cat": "charent/self_cognition_Bob",  # same as Bob
+}
+
+LORA_RANK = 8
+
+LORA_TEST_PROMPTS = ["What is GitHub?", "Hi, tell me about you"]
+LORA_TEST_EXPECTED = [
+    "GitHub is an open-source platform that provides a way to manage and develop software projects. It allows developers to store and manage code, collaborate on projects, and automate tasks.",  # noqa: E501
+    "I am Alice, an AI assistant developed by GitHub/Charent.",  # noqa: E501
+]
+
+
+def format_chatml_messages(prompt: str):
+    return [
+        {"role": "system", "content": "You are a helpful assistant."},
+        {"role": "user", "content": prompt},
+    ]
+
+
+@patch.dict("os.environ", {"VLLM_USE_MODELSCOPE": "False"})
+def test_multi_loras_with_tp_sync():
+    lora_name_id_map = {}
+    increase_lora_id = 0
+
+    def make_add_lora_request(name: str, path: str):
+        nonlocal increase_lora_id
+        increase_lora_id += 1
+        lora_name_id_map[name] = increase_lora_id
+
+        return LoRARequest(
+            lora_name=name,
+            lora_int_id=increase_lora_id,
+            lora_path=path,
+        )
+
+    vllm_model = VllmRunner(
+        MODEL_PATH,
+        enable_lora=True,
+        # dtype="half",
+        max_loras=2,  # ensure max_loras < max_cpu_loras
+        max_lora_rank=LORA_RANK,
+        max_model_len=512,
+        gpu_memory_utilization=0.9,
+        enforce_eager=True,
+        # tensor_parallel_size=2,  # ensure tp >= 2
+        max_cpu_loras=4,  # ensure max_cpu_loras >= 2
+    )
+    llm = vllm_model.model
+
+    def run_check_lora(fn, args, expected: list):
+        fn(args)
+        assert set(llm.llm_engine.list_loras()) == set(expected)
+
+    # simulate add loras with CLI args
+    # likes: `--lora-modules Alice=/path/to/Alice Bob=/path/to/Bob`
+    run_check_lora(
+        llm.llm_engine.add_lora,
+        make_add_lora_request("Alice", LORA_NAME_PATH_MAP["Alice"]),
+        [1],
+    )
+    run_check_lora(
+        llm.llm_engine.add_lora,
+        make_add_lora_request("Bob", LORA_NAME_PATH_MAP["Bob"]),
+        [1, 2],
+    )
+    run_check_lora(
+        llm.llm_engine.add_lora,
+        make_add_lora_request("Cat", LORA_NAME_PATH_MAP["Cat"]),
+        [1, 2, 3],
+    )
+
+    # set temperature = 0 for greedy search
+    sampling_params = SamplingParams(temperature=0, max_tokens=64)
+
+    def call_llm_get_outputs(prompt: str, lora_name: str):
+        lora_request = LoRARequest(
+            lora_name=lora_name,
+            lora_int_id=lora_name_id_map[lora_name],
+            lora_path=LORA_NAME_PATH_MAP[lora_name],
+        )
+        messages = format_chatml_messages(prompt)
+        outputs = llm.chat(
+            [messages],
+            sampling_params,
+            chat_template_kwargs={"enable_thinking": False},  # for those loras, ensure enable_thinking=False
+            lora_request=lora_request,
+            use_tqdm=False,
+        )
+        output_text = outputs[0].outputs[0].text
+        return output_text
+
+    def reload_lora(name: str):
+        """
+        reload a lora to simulate the case:
+        setting `VLLM_ALLOW_RUNTIME_LORA_UPDATING=true`
+        for dynamic lora loading and unloading
+        """
+        remove_lora_response = llm.llm_engine.remove_lora(lora_id=lora_name_id_map[name])
+
+        add_lora_response = llm.llm_engine.add_lora(make_add_lora_request(name, LORA_NAME_PATH_MAP[name]))
+
+        print(f"{remove_lora_response=}, {add_lora_response=}")
+
+    def check_outputs(outputs: str, expected: str, prompt: str):
+        print(f"{prompt=}.\n{expected=}\n{outputs=}")
+        print("\n----------------------------\n")
+        assert outputs == expected
+
+    for prompt, expected_output in zip(LORA_TEST_PROMPTS, LORA_TEST_EXPECTED):
+        output_text = call_llm_get_outputs(prompt, "Alice")
+        check_outputs(output_text, expected_output, prompt)
+
+        # call Bob, ignore what it is output
+        call_llm_get_outputs(prompt, "Bob")
+        print("After call Bob:")
+
+        # call Alice
+        output_text = call_llm_get_outputs(prompt, "Alice")
+        check_outputs(output_text, expected_output, prompt)
+
+        # reload Bob Lora
+        reload_lora("Bob")
+        print("After reload Bob:")
+
+        # call Alice
+        output_text = call_llm_get_outputs(prompt, "Alice")
+        check_outputs(output_text, expected_output, prompt)
+
+        # reload Alice Lora
+        reload_lora("Alice")
+        print("After reload Alice:")
+
+        output_text = call_llm_get_outputs(prompt, "Alice")
+        check_outputs(output_text, expected_output, prompt)

tests/e2e/pull_request/one_card/lora/test_qwen3_reranker_lora.py

@@ -0,0 +1,74 @@
+from pathlib import Path
+
+from vllm import LLM
+
+model_name = "Qwen/Qwen3-Reranker-0.6B"
+
+
+def get_llm() -> LLM:
+    """
+    Initializes and returns the LLM model for Qwen3-Reranker.
+
+    Returns:
+        LLM: Configured vLLM instance for reranking tasks.
+
+    Note:
+        This function loads the ORIGINAL Qwen3-Reranker model with specific
+        overrides to make it compatible with vLLM's score API.
+    """
+    return LLM(
+        # Specify the original model from HuggingFace
+        model=model_name,
+        # Use pooling runner for score task
+        runner="pooling",
+        # HuggingFace model configuration overrides required for compatibility
+        hf_overrides={
+            # Manually route to sequence classification architecture
+            # This tells vLLM to use Qwen3ForSequenceClassification instead of
+            # the default Qwen3ForCausalLM
+            "architectures": ["Qwen3ForSequenceClassification"],
+            # Specify which token logits to extract from the language model head
+            # The original reranker uses "no" and "yes" token logits for scoring
+            "classifier_from_token": ["no", "yes"],
+            # Enable special handling for original Qwen3-Reranker models
+            # This flag triggers conversion logic that transforms the two token
+            # vectors into a single classification vector
+            "is_original_qwen3_reranker": True,
+        },
+        enable_lora=True,
+    )
+
+
+def test_reranker_models_lora():
+    # Load the Jinja template for formatting query-document pairs
+    # The template ensures proper formatting for the reranker model
+    template_home = Path(__file__).parents[1] / "pooling" / "template"
+    template_path = "qwen3_reranker.jinja"
+    chat_template = (template_home / template_path).read_text()
+
+    # Sample queries for testing the reranker
+    queries = [
+        "What is the capital of China?",
+        "Explain gravity",
+    ]
+
+    # Corresponding documents to be scored against each query
+    documents = [
+        "The capital of China is Beijing.",
+        "Gravity is a force that attracts two bodies towards each other. It gives weight to physical objects and is "
+        "responsible for the movement of planets around the sun.",
+    ]
+
+    # Initialize the LLM model with the original Qwen3-Reranker configuration
+    llm = get_llm()
+
+    # Compute relevance scores for each query-document pair
+    # The score() method returns a relevance score for each pair
+    # Higher scores indicate better relevance
+    outputs = llm.score(queries, documents, chat_template=chat_template)
+
+    # Extract and print the relevance scores from the outputs
+    # Each output contains a score representing query-document relevance
+    print("-" * 30)
+    print("Relevance scores:", [output.outputs.score for output in outputs])
+    print("-" * 30)