init v0.23.0

Signed-off-by: Sun Ruoxi <sunruoxi@4paradigm.com>

tests/e2e/pull_request/one_card/pooling/template/qwen3_reranker.jinja

@@ -0,0 +1,11 @@
+<|im_start|>system
+Judge whether the Document meets the requirements based on the Query and the Instruct provided. Note that the answer can only be "yes" or "no".<|im_end|>
+<|im_start|>user
+<Instruct>: {{ messages | selectattr("role", "eq", "system") | map(attribute="content") | first | default("Given a web search query, retrieve relevant passages that answer the query") }}
+<Query>: {{ messages | selectattr("role", "eq", "query") | map(attribute="content") | first }}
+<Document>: {{ messages | selectattr("role", "eq", "document") | map(attribute="content") | first }}<|im_end|>
+<|im_start|>assistant
+<think>
+
+</think>
+

tests/e2e/pull_request/one_card/pooling/test_classification.py

@@ -0,0 +1,43 @@
+import huggingface_hub
+import torch
+from modelscope import snapshot_download  # type: ignore[import-untyped]
+from transformers import AutoModelForSequenceClassification
+
+from tests.e2e.conftest import (
+    HfRunner,
+    VllmRunner,
+    cleanup_dist_env_and_memory,
+    wait_until_npu_memory_free,
+)
+
+
+@wait_until_npu_memory_free(target_free_percentage=0.7)
+def test_qwen_pooling_classify_correctness() -> None:
+    model_name = snapshot_download(
+        "Howeee/Qwen2.5-1.5B-apeach",
+        local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
+    )
+
+    prompts = [
+        "Hello, my name is",
+        "The president of the United States is",
+        "The capital of France is",
+        "The future of AI is what",
+    ]
+
+    with HfRunner(model_name, dtype="float32", auto_cls=AutoModelForSequenceClassification) as hf_runner:
+        hf_outputs = hf_runner.classify(prompts)
+    cleanup_dist_env_and_memory()
+
+    with VllmRunner(
+        model_name,
+        runner="pooling",
+        max_model_len=None,
+        cudagraph_capture_sizes=[4],
+    ) as vllm_runner:
+        vllm_outputs = vllm_runner.classify(prompts)
+
+    for hf_output, vllm_output in zip(hf_outputs, vllm_outputs):
+        hf_output = torch.tensor(hf_output)
+        vllm_output = torch.tensor(vllm_output)
+        assert torch.allclose(hf_output, vllm_output, 1e-2)

tests/e2e/pull_request/one_card/pooling/test_embedding.py

@@ -0,0 +1,135 @@
+#
+# Copyright (c) 2025 Huawei Technologies Co., Ltd. All Rights Reserved.
+# Copyright 2023 The vLLM team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# This file is a part of the vllm-ascend project.
+# Adapted from vllm/tests/basic_correctness/test_basic_correctness.py
+#
+import huggingface_hub
+import pytest
+from modelscope import snapshot_download  # type: ignore[import-untyped]
+
+from tests.e2e.conftest import HfRunner, VllmRunner
+from tests.e2e.utils import check_embeddings_close
+
+MODELS = [
+    "Qwen/Qwen3-Embedding-0.6B",  # lasttoken
+    "intfloat/multilingual-e5-small",  # mean_tokens
+]
+
+
+@pytest.mark.parametrize("model", MODELS)
+def test_embed_models_correctness(model: str):
+    queries = ["What is the capital of China?", "Explain gravity"]
+
+    model_name = snapshot_download(
+        model,
+        local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
+    )
+    with VllmRunner(
+        model_name,
+        runner="pooling",
+        max_model_len=None,
+        cudagraph_capture_sizes=[4],
+    ) as vllm_runner:
+        vllm_outputs = vllm_runner.embed(queries)
+
+    with HfRunner(
+        model_name,
+        dtype="float32",
+        is_sentence_transformer=True,
+    ) as hf_runner:
+        hf_outputs = hf_runner.encode(queries)
+
+    check_embeddings_close(
+        embeddings_0_lst=hf_outputs,
+        embeddings_1_lst=vllm_outputs,
+        name_0="hf",
+        name_1="vllm",
+        tol=1e-2,
+    )
+
+
+def test_causal_embed_models_using_prefix_caching_correctness():
+    # This test is to verify the correctness of prefix caching for embedding models.
+    # We compare the outputs of vLLM with and without prefix caching enabled, and check if they are close enough.
+    # We set the input query to be very long to make sure prefix caching is triggered.
+    queries = ["What is the capital of China?" * 256, "Explain gravity"]
+
+    model_name = snapshot_download(
+        "Qwen/Qwen3-Embedding-0.6B",
+        local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
+    )
+    with VllmRunner(
+        model_name,
+        runner="pooling",
+        max_model_len=None,
+        cudagraph_capture_sizes=[4],
+        enable_prefix_caching=True,
+    ) as vllm_runner_using_caching:
+        vllm_outputs_without_caching = vllm_runner_using_caching.embed(queries)
+        vllm_outputs_with_caching = vllm_runner_using_caching.embed(queries)
+
+    check_embeddings_close(
+        embeddings_0_lst=vllm_outputs_without_caching,
+        embeddings_1_lst=vllm_outputs_with_caching,
+        name_0="without_caching",
+        name_1="with_caching",
+        tol=1e-2,
+    )
+
+
+def test_bge_m3_correctness():
+    queries = ["What is the capital of China?", "Explain gravity"]
+
+    model_name = snapshot_download(
+        "BAAI/bge-m3",
+        local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
+    )
+    with VllmRunner(
+        model_name,
+        runner="pooling",
+        cudagraph_capture_sizes=[4],
+    ) as vllm_aclgraph_runner:
+        vllm_aclgraph_outputs = vllm_aclgraph_runner.embed(queries)
+
+    with VllmRunner(
+        model_name,
+        runner="pooling",
+        enforce_eager=True,
+    ) as vllm_runner:
+        vllm_eager_outputs = vllm_runner.embed(queries)
+
+    with HfRunner(
+        model_name,
+        dtype="float32",
+        is_sentence_transformer=True,
+    ) as hf_runner:
+        hf_outputs = hf_runner.encode(queries)
+
+    check_embeddings_close(
+        embeddings_0_lst=hf_outputs,
+        embeddings_1_lst=vllm_eager_outputs,
+        name_0="hf",
+        name_1="vllm",
+        tol=1e-2,
+    )
+
+    check_embeddings_close(
+        embeddings_0_lst=vllm_eager_outputs,
+        embeddings_1_lst=vllm_aclgraph_outputs,
+        name_0="eager",
+        name_1="aclgraph",
+        tol=1e-2,
+    )

tests/e2e/pull_request/one_card/pooling/test_scoring.py

@@ -0,0 +1,167 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+import huggingface_hub
+import pytest
+import torch
+import torch.nn.functional as F
+from modelscope import snapshot_download  # type: ignore[import-untyped]
+
+from tests.e2e.conftest import HfRunner, VllmRunner
+
+CROSS_ENCODER_MODELS = [
+    "dengcao/ms-marco-MiniLM-L6-v2",  # Bert
+    "BAAI/bge-reranker-v2-m3",  # Roberta
+]
+
+EMBEDDING_MODELS = [
+    "sentence-transformers/all-MiniLM-L12-v2",
+]
+
+TEXTS_1 = [
+    "What is the capital of France?",
+    "What is the capital of Germany?",
+]
+
+TEXTS_2 = [
+    "The capital of France is Paris.",
+    "The capital of Germany is Berlin.",
+]
+
+DTYPE = "half"
+
+
+@pytest.fixture(scope="module", params=CROSS_ENCODER_MODELS)
+def model_name(request):
+    yield snapshot_download(
+        request.param,
+        local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
+    )
+
+
+def test_cross_encoder_score_1_to_1(model_name):
+    text_pair = [TEXTS_1[0], TEXTS_2[0]]
+
+    with HfRunner(model_name, dtype=DTYPE, is_cross_encoder=True) as hf_model:
+        hf_outputs = hf_model.predict([text_pair]).tolist()
+
+    with VllmRunner(
+        model_name, runner="pooling", dtype=DTYPE, cudagraph_capture_sizes=[4], max_model_len=None
+    ) as vllm_model:
+        vllm_outputs = vllm_model.score(text_pair[0], text_pair[1])
+
+    assert len(vllm_outputs) == 1
+    assert len(hf_outputs) == 1
+
+    assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
+
+
+def test_cross_encoder_score_1_to_N(model_name):
+    text_pairs = [
+        [TEXTS_1[0], TEXTS_2[0]],
+        [TEXTS_1[0], TEXTS_2[1]],
+    ]
+
+    with HfRunner(model_name, dtype=DTYPE, is_cross_encoder=True) as hf_model:
+        hf_outputs = hf_model.predict(text_pairs).tolist()
+
+    with VllmRunner(
+        model_name, runner="pooling", dtype=DTYPE, cudagraph_capture_sizes=[4], max_model_len=None
+    ) as vllm_model:
+        vllm_outputs = vllm_model.score(TEXTS_1[0], TEXTS_2)
+
+    assert len(vllm_outputs) == 2
+    assert len(hf_outputs) == 2
+
+    assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
+    assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
+
+
+def test_cross_encoder_score_N_to_N(model_name):
+    text_pairs = [
+        [TEXTS_1[0], TEXTS_2[0]],
+        [TEXTS_1[1], TEXTS_2[1]],
+    ]
+
+    with HfRunner(model_name, dtype=DTYPE, is_cross_encoder=True) as hf_model:
+        hf_outputs = hf_model.predict(text_pairs).tolist()
+
+    with VllmRunner(
+        model_name, runner="pooling", dtype=DTYPE, cudagraph_capture_sizes=[4], max_model_len=None
+    ) as vllm_model:
+        vllm_outputs = vllm_model.score(TEXTS_1, TEXTS_2)
+
+    assert len(vllm_outputs) == 2
+    assert len(hf_outputs) == 2
+
+    assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
+    assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
+
+
+@pytest.fixture(scope="module", params=EMBEDDING_MODELS)
+def emb_model_name(request):
+    yield snapshot_download(
+        request.param,
+        local_files_only=huggingface_hub.constants.HF_HUB_OFFLINE,
+    )
+
+
+def test_embedding_score_1_to_1(emb_model_name):
+    text_pair = [TEXTS_1[0], TEXTS_2[0]]
+
+    with HfRunner(emb_model_name, dtype=DTYPE, is_sentence_transformer=True) as hf_model:
+        hf_embeddings = hf_model.encode(text_pair)
+        hf_outputs = [F.cosine_similarity(*map(torch.tensor, hf_embeddings), dim=0)]
+
+    with VllmRunner(
+        emb_model_name, runner="pooling", dtype=DTYPE, cudagraph_capture_sizes=[4], max_model_len=None
+    ) as vllm_model:
+        vllm_outputs = vllm_model.score(text_pair[0], text_pair[1])
+
+    assert len(vllm_outputs) == 1
+    assert len(hf_outputs) == 1
+
+    assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
+
+
+def test_embedding_score_1_to_N(emb_model_name):
+    text_pairs = [
+        [TEXTS_1[0], TEXTS_2[0]],
+        [TEXTS_1[0], TEXTS_2[1]],
+    ]
+
+    with HfRunner(emb_model_name, dtype=DTYPE, is_sentence_transformer=True) as hf_model:
+        hf_embeddings = [hf_model.encode(text_pair) for text_pair in text_pairs]
+        hf_outputs = [F.cosine_similarity(*map(torch.tensor, pair), dim=0) for pair in hf_embeddings]
+
+    with VllmRunner(
+        emb_model_name, runner="pooling", dtype=DTYPE, cudagraph_capture_sizes=[4], max_model_len=None
+    ) as vllm_model:
+        vllm_outputs = vllm_model.score(TEXTS_1[0], TEXTS_2)
+
+    assert len(vllm_outputs) == 2
+    assert len(hf_outputs) == 2
+
+    assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
+    assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)
+
+
+def test_embedding_score_N_to_N(emb_model_name):
+    text_pairs = [
+        [TEXTS_1[0], TEXTS_2[0]],
+        [TEXTS_1[1], TEXTS_2[1]],
+    ]
+
+    with HfRunner(emb_model_name, dtype=DTYPE, is_sentence_transformer=True) as hf_model:
+        hf_embeddings = [hf_model.encode(text_pair) for text_pair in text_pairs]
+        hf_outputs = [F.cosine_similarity(*map(torch.tensor, pair), dim=0) for pair in hf_embeddings]
+
+    with VllmRunner(
+        emb_model_name, runner="pooling", dtype=DTYPE, cudagraph_capture_sizes=[4], max_model_len=None
+    ) as vllm_model:
+        vllm_outputs = vllm_model.score(TEXTS_1, TEXTS_2)
+
+    assert len(vllm_outputs) == 2
+    assert len(hf_outputs) == 2
+
+    assert hf_outputs[0] == pytest.approx(vllm_outputs[0], rel=0.01)
+    assert hf_outputs[1] == pytest.approx(vllm_outputs[1], rel=0.01)