init v0.23.0

Signed-off-by: Sun Ruoxi <sunruoxi@4paradigm.com>

tests/e2e/models/configs/ERNIE-4.5-21B-A3B-PT.yaml

@@ -0,0 +1,21 @@
+model_name: "PaddlePaddle/ERNIE-4.5-21B-A3B-PT"
+model_type: "vllm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.8
+  trust_remote_code: true
+
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,flexible-extract"
+    value: 0.71
+
+num_fewshot: 5
+apply_chat_template: true
+fewshot_as_multiturn: true
+batch_size: "auto"

tests/e2e/models/configs/Hunyuan-A13B-Instruct.yaml

@@ -0,0 +1,25 @@
+model_name: "Tencent-Hunyuan/Hunyuan-A13B-Instruct"
+model_type: "vllm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 4
+  dtype: auto
+  max_model_len: 32768
+  gpu_memory_utilization: 0.90
+  enforce_eager: true
+  trust_remote_code: true
+
+tasks:
+  - name: "gsm8k"
+    metrics:
+      - name: "exact_match,strict-match"
+        value: 0.37
+      - name: "exact_match,flexible-extract"
+        value: 0.28
+
+num_fewshot: 5
+limit: 1000
+apply_chat_template: false
+fewshot_as_multiturn: false
+batch_size: "auto"

tests/e2e/models/configs/InternVL3_5-8B-hf.yaml

@@ -0,0 +1,21 @@
+model_name: "OpenGVLab/InternVL3_5-8B-hf"
+model_type: "vllm-vlm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 40960
+  gpu_memory_utilization: 0.8
+  trust_remote_code: true
+
+tasks:
+  - name: "mmmu_val"
+    metrics:
+    - name: "acc,none"
+      value: 0.58
+
+num_fewshot: 0
+apply_chat_template: false
+fewshot_as_multiturn: false
+batch_size: "auto"

tests/e2e/models/configs/Llama-3.2-3B-Instruct.yaml

@@ -0,0 +1,23 @@
+model_name: "LLM-Research/Llama-3.2-3B-Instruct"
+model_type: "vllm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.8
+  trust_remote_code: false
+
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,strict-match"
+    value: 0.71
+  - name: "exact_match,flexible-extract"
+    value: 0.76
+
+num_fewshot: 5
+apply_chat_template: true
+fewshot_as_multiturn: true
+batch_size: "auto"

tests/e2e/models/configs/Minitron-8B-Base.yaml

@@ -0,0 +1,24 @@
+model_name: "nv-community/Minitron-8B-Base"
+model_type: "vllm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.9
+  enforce_eager: true
+  trust_remote_code: true
+
+tasks:
+  - name: "gsm8k"
+    metrics:
+      - name: "exact_match,strict-match"
+        value: 0.5436
+      - name: "exact_match,flexible-extract"
+        value: 0.5451
+
+limit: 1000
+apply_chat_template: false
+fewshot_as_multiturn: false
+batch_size: "auto"

tests/e2e/models/configs/Mixtral-8x7B-Instruct-v0.1.yaml

@@ -0,0 +1,32 @@
+model_name: "mistralai/Mixtral-8x7B-Instruct-v0.1"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 4
+  dtype: bfloat16
+  max_model_len: 4096
+  gpu_memory_utilization: 0.7
+  trust_remote_code: true
+  enforce_eager: true
+  block_size: 128
+
+envs:
+  HCCL_OP_EXPANSION_MODE: "AIV"
+  OMP_PROC_BIND: "false"
+  OMP_NUM_THREADS: "10"
+  VLLM_USE_V1: "1"
+  HCCL_BUFFSIZE: "200"
+  VLLM_ASCEND_ENABLE_MLAPO: "1"
+  PYTORCH_NPU_ALLOC_CONF: "expandable_segments:True"
+  VLLM_ASCEND_ENABLE_FLASHCOMM1: "1"
+
+tasks:
+  - name: "ceval-valid"
+    metrics:
+      - name: "acc,none"
+        value: 0.45
+
+num_fewshot: 5
+apply_chat_template: true
+fewshot_as_multiturn: false
+batch_size: 32

tests/e2e/models/configs/Molmo-7B-D-0924.yaml

@@ -0,0 +1,21 @@
+model_name: "LLM-Research/Molmo-7B-D-0924"
+model_type: "vllm-vlm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.8
+  trust_remote_code: true
+
+tasks:
+- name: "ceval-valid"
+  metrics:
+  - name: "acc,none"
+    value: 0.71
+
+num_fewshot: 0
+apply_chat_template: false
+fewshot_as_multiturn: false
+batch_size: "auto"

tests/e2e/models/configs/Qwen2-Audio-7B-Instruct.yaml

@@ -0,0 +1,23 @@
+model_name: "Qwen/Qwen2-Audio-7B-Instruct"
+model_type: "vllm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.8
+  trust_remote_code: false
+
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,strict-match"
+    value: 0.44
+  - name: "exact_match,flexible-extract"
+    value: 0.45
+
+num_fewshot: 5
+apply_chat_template: true
+fewshot_as_multiturn: true
+batch_size: "auto"

tests/e2e/models/configs/Qwen2.5-Math-RM-72B.yaml

@@ -0,0 +1,25 @@
+model_name: "Qwen/Qwen2.5-Math-RM-72B"
+model_type: "vllm-rm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 4
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.9
+  trust_remote_code: false
+
+# system_prompt controls the <|im_start|>system block passed to the reward model.
+system_prompt: "Please reason step by step, and put your final answer within \\boxed{}."
+
+tasks:
+  - name: "gsm8k_correctness"
+    dataset: "AI-ModelScope/gsm8k"
+    split: "test"
+    dataset_config: "main"
+    metrics:
+      - name: "accuracy"
+        value: 0.80
+
+limit: 200
+batch_size: 4

tests/e2e/models/configs/Qwen3-30B-A3B-W8A8.yaml

@@ -0,0 +1,25 @@
+model_name: "vllm-ascend/Qwen3-30B-A3B-W8A8"
+model_type: "vllm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 2
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.7
+  trust_remote_code: false
+  enable_expert_parallel: true
+  quantization: ascend
+
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,strict-match"
+    value: 0.9
+  - name: "exact_match,flexible-extract"
+    value: 0.8
+
+num_fewshot: 5
+apply_chat_template: false
+fewshot_as_multiturn: false
+batch_size: "auto"

tests/e2e/models/configs/Qwen3-30B-A3B.yaml

@@ -1,6 +1,15 @@
 model_name: "Qwen/Qwen3-30B-A3B"
-runner: "linux-aarch64-a2-2"
+model_type: "vllm"
 hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 2
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.6
+  trust_remote_code: false
+  enable_expert_parallel: true
+
 tasks:
 - name: "gsm8k"
   metrics:
@@ -12,9 +21,8 @@ tasks:
   metrics:
   - name: "acc,none"
     value: 0.84
+
 num_fewshot: 5
-gpu_memory_utilization: 0.6
-enable_expert_parallel: True
-tensor_parallel_size: 2
-apply_chat_template: False
-fewshot_as_multiturn: False
+apply_chat_template: false
+fewshot_as_multiturn: false
+batch_size: "auto"

tests/e2e/models/configs/Qwen3-8B-W8A8.yaml

@@ -0,0 +1,25 @@
+model_name: "vllm-ascend/Qwen3-8B-W8A8"
+model_type: "vllm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.8
+  trust_remote_code: false
+  quantization: ascend
+  enable_thinking: false
+
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,strict-match"
+    value: 0.80
+  - name: "exact_match,flexible-extract"
+    value: 0.82
+
+num_fewshot: 5
+apply_chat_template: true
+fewshot_as_multiturn: true
+batch_size: "auto"

tests/e2e/models/configs/Qwen3-8B.yaml

@@ -0,0 +1,24 @@
+model_name: "Qwen/Qwen3-8B"
+model_type: "vllm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.8
+  trust_remote_code: false
+  enable_thinking: false
+
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,strict-match"
+    value: 0.765
+  - name: "exact_match,flexible-extract"
+    value: 0.81
+
+num_fewshot: 5
+apply_chat_template: true
+fewshot_as_multiturn: true
+batch_size: "auto"

tests/e2e/models/configs/Qwen3-ASR-1.7B.yaml

@@ -0,0 +1,21 @@
+model_name: "Qwen/Qwen3-ASR-1.7B"
+model_type: "vllm-asr"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.8
+  trust_remote_code: false
+
+tasks:
+  - name: "librispeech_test_clean"
+    dataset: "openslr/librispeech_asr"
+    split: "test"
+    dataset_config: "clean"
+    metrics:
+      - name: "wer"
+        value: 0.035
+
+limit: 500

tests/e2e/models/configs/Qwen3-Next-80B-A3B-Instruct.yaml

@@ -0,0 +1,23 @@
+model_name: "Qwen/Qwen3-Next-80B-A3B-Instruct"
+model_type: "vllm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 4
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.7
+  trust_remote_code: false
+  enable_expert_parallel: true
+  enforce_eager: true
+
+tasks:
+- name: "ceval-valid_accountant"
+  metrics:
+  - name: "acc,none"
+    value: 0.98
+
+num_fewshot: 5
+apply_chat_template: true
+fewshot_as_multiturn: true
+batch_size: 1

tests/e2e/models/configs/Qwen3-Omni-30B-A3B-Instruct.yaml

@@ -0,0 +1,22 @@
+model_name: "Qwen/Qwen3-Omni-30B-A3B-Instruct"
+model_type: "vllm-vlm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 4
+  dtype: auto
+  max_model_len: 8192
+  gpu_memory_utilization: 0.7
+  trust_remote_code: false
+  enable_expert_parallel: true
+
+tasks:
+- name: "mmmu_val"
+  metrics:
+  - name: "acc,none"
+    value: 0.60
+
+num_fewshot: 0
+apply_chat_template: true
+fewshot_as_multiturn: false
+batch_size: "auto"

tests/e2e/models/configs/Qwen3-VL-30B-A3B-Instruct.yaml

@@ -0,0 +1,22 @@
+model_name: "Qwen/Qwen3-VL-30B-A3B-Instruct"
+model_type: "vllm-vlm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 2
+  dtype: auto
+  max_model_len: 128000
+  gpu_memory_utilization: 0.7
+  trust_remote_code: false
+  enable_expert_parallel: true
+
+tasks:
+- name: "mmmu_val"
+  metrics:
+  - name: "acc,none"
+    value: 0.58
+
+num_fewshot: 0
+apply_chat_template: true
+fewshot_as_multiturn: false
+batch_size: "auto"

tests/e2e/models/configs/Qwen3-VL-8B-Instruct-W8A8.yaml

@@ -0,0 +1,22 @@
+model_name: "vllm-ascend/Qwen3-VL-8B-Instruct-W8A8"
+model_type: "vllm-vlm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 8192
+  gpu_memory_utilization: 0.8
+  trust_remote_code: false
+  quantization: ascend
+
+tasks:
+- name: "mmmu_val"
+  metrics:
+  - name: "acc,none"
+    value: 0.52
+
+num_fewshot: 0
+apply_chat_template: true
+fewshot_as_multiturn: false
+batch_size: 32

tests/e2e/models/configs/Qwen3-VL-8B-Instruct.yaml

@@ -0,0 +1,21 @@
+model_name: "Qwen/Qwen3-VL-8B-Instruct"
+model_type: "vllm-vlm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 8192
+  gpu_memory_utilization: 0.7
+  trust_remote_code: false
+
+tasks:
+- name: "mmmu_val"
+  metrics:
+  - name: "acc,none"
+    value: 0.55
+
+num_fewshot: 0
+apply_chat_template: true
+fewshot_as_multiturn: false
+batch_size: 32

tests/e2e/models/configs/accuracy.txt

@@ -1,4 +1,14 @@
-DeepSeek-V2-Lite.yaml
-Qwen3-8B-Base.yaml
-Qwen2.5-VL-7B-Instruct.yaml
-Qwen3-30B-A3B.yaml
+Qwen3-30B-A3B.yaml
+Qwen3-8B.yaml
+Qwen2-Audio-7B-Instruct.yaml
+Qwen3-VL-30B-A3B-Instruct.yaml
+Qwen3-VL-8B-Instruct.yaml
+Qwen3-Omni-30B-A3B-Instruct.yaml
+InternVL3_5-8B-hf.yaml
+ERNIE-4.5-21B-A3B-PT.yaml
+gemma-3-4b-it.yaml
+internlm3-8b-instruct.yaml
+Molmo-7B-D-0924.yaml
+llava-onevision-qwen2-0.5b-ov-hf.yaml
+Llama-3.2-3B-Instruct.yaml
+Qwen3-ASR-1.7B.yaml

tests/e2e/models/configs/gemma-3-4b-it.yaml

@@ -0,0 +1,24 @@
+model_name: "LLM-Research/gemma-3-4b-it"
+model_type: "vllm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.7
+  trust_remote_code: false
+  enforce_eager: true
+
+tasks:
+- name: "gsm8k"
+  metrics:
+  - name: "exact_match,strict-match"
+    value: 0.59
+  - name: "exact_match,flexible-extract"
+    value: 0.59
+
+num_fewshot: 5
+apply_chat_template: false
+fewshot_as_multiturn: false
+batch_size: "auto"

tests/e2e/models/configs/internlm3-8b-instruct.yaml

@@ -0,0 +1,21 @@
+model_name: "Shanghai_AI_Laboratory/internlm3-8b-instruct"
+model_type: "vllm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: "bfloat16"
+  max_model_len: 2048
+  gpu_memory_utilization: 0.8
+  trust_remote_code: true
+
+tasks:
+- name: "ceval-valid"
+  metrics:
+  - name: "acc,none"
+    value: 0.42
+
+num_fewshot: 5
+apply_chat_template: false
+fewshot_as_multiturn: false
+batch_size: "auto"

tests/e2e/models/configs/llava-onevision-qwen2-0.5b-ov-hf.yaml

@@ -0,0 +1,21 @@
+model_name: "llava-hf/llava-onevision-qwen2-0.5b-ov-hf"
+model_type: "vllm-vlm"
+hardware: "Atlas A2 Series"
+
+serve:
+  tensor_parallel_size: 1
+  dtype: auto
+  max_model_len: 4096
+  gpu_memory_utilization: 0.8
+  trust_remote_code: true
+
+tasks:
+- name: "ceval-valid"
+  metrics:
+  - name: "acc,none"
+    value: 0.42
+
+num_fewshot: 0
+apply_chat_template: false
+fewshot_as_multiturn: false
+batch_size: "auto"

tests/e2e/models/conftest.py

@@ -21,7 +21,7 @@ def pytest_addoption(parser):
     parser.addoption(
         "--config",
         action="store",
-        default="./tests/e2e/models/configs/Qwen3-8B-Base.yaml",
+        default="./tests/e2e/models/configs/Qwen3-8B.yaml",
         help="Path to the model config YAML file",
     )
     parser.addoption(
@@ -55,16 +55,12 @@ def report_dir(pytestconfig):
 
 def pytest_generate_tests(metafunc):
     if "config_filename" in metafunc.fixturenames:
-
         if metafunc.config.getoption("--config-list-file"):
             rel_path = metafunc.config.getoption("--config-list-file")
             config_list_file = Path(rel_path).resolve()
             config_dir = config_list_file.parent
             with open(config_list_file, encoding="utf-8") as f:
-                configs = [
-                    config_dir / line.strip() for line in f
-                    if line.strip() and not line.startswith("#")
-                ]
+                configs = [config_dir / line.strip() for line in f if line.strip() and not line.startswith("#")]
             metafunc.parametrize("config_filename", configs)
         else:
             single_config = metafunc.config.getoption("--config")

tests/e2e/models/report_template.md

@@ -1,30 +1,33 @@
 # {{ model_name }}
 
 - **vLLM Version**: vLLM: {{ vllm_version }} ([{{ vllm_commit[:7] }}](https://github.com/vllm-project/vllm/commit/{{ vllm_commit }})), **vLLM Ascend Version**: {{ vllm_ascend_version }} ([{{ vllm_ascend_commit[:7] }}](https://github.com/vllm-project/vllm-ascend/commit/{{ vllm_ascend_commit }}))  
-- **Software Environment**: **CANN**: {{ cann_version }}, **PyTorch**: {{ torch_version }}, **torch-npu**: {{ torch_npu_version }}  
+- **Software Environment**: **CANN**: {{ cann_version }}, **PyTorch**: {{ torch_version }}, **TorchNPU**: {{ torch_npu_version }}  
 - **Hardware Environment**: {{ hardware }}
 - **Parallel mode**: {{ parallel_mode }}
 - **Execution mode**: {{ execution_model }}
 
+{% if show_command is not defined or show_command %}
 **Command**:  
 
 ```bash
 export MODEL_ARGS={{ model_args }}
-lm_eval --model {{ model_type }} --model_args $MODEL_ARGS --tasks {{ datasets }} \
-{% if apply_chat_template is defined and (apply_chat_template|string|lower in ["true", "1"]) -%}
+lm_eval --model {{ model_type }} --model_args $MODEL_ARGS \
+  --tasks {{ datasets }} \
+{%- if apply_chat_template is defined and (apply_chat_template|string|lower in ["true", "1"]) %}
   --apply_chat_template \
 {%- endif %}
-{% if fewshot_as_multiturn is defined and (fewshot_as_multiturn|string|lower in ["true", "1"]) -%}
+{%- if fewshot_as_multiturn is defined and (fewshot_as_multiturn|string|lower in ["true", "1"]) %}
   --fewshot_as_multiturn \
 {%- endif %}
-{% if num_fewshot is defined and num_fewshot != "N/A" -%}
+{%- if num_fewshot is defined and num_fewshot != "N/A" %}
   --num_fewshot {{ num_fewshot }} \
 {%- endif %}
-{% if limit is defined and limit != "N/A" -%}
+{%- if limit is defined and limit != "N/A" %}
   --limit {{ limit }} \
 {%- endif %}
---batch_size {{ batch_size }}
+  --batch_size {{ batch_size }}
 ```
+{% endif %}
 
 | Task                  | Metric      | Value     | Stderr |
 |-----------------------|-------------|----------:|-------:|

tests/e2e/models/test_asr_eval_correctness.py

@@ -0,0 +1,290 @@
+import io
+import os
+import string
+from dataclasses import dataclass
+
+import jiwer  # type: ignore[import-untyped]
+import numpy as np
+import pytest
+import scipy.io.wavfile as wav_io  # type: ignore[import-untyped]
+import soundfile as sf  # type: ignore[import-untyped]
+import yaml
+from datasets import Audio
+from jinja2 import Environment, FileSystemLoader
+from modelscope.msdatasets import MsDataset  # type: ignore[import-untyped]
+from vllm.utils.network_utils import get_open_port
+
+from tests.e2e.conftest import RemoteOpenAIServer
+
+# Allow up to 10% relative deviation from the declared ground-truth WER.
+# ASR results have higher variance than classification tasks, so we use a
+# more generous tolerance than the 5% used in test_lm_eval_correctness.py.
+RTOL = 0.03
+
+TEST_DIR = os.path.dirname(__file__)
+
+_PUNCT_TABLE = str.maketrans("", "", string.punctuation)
+
+
+@dataclass
+class EnvConfig:
+    vllm_version: str
+    vllm_commit: str
+    vllm_ascend_version: str
+    vllm_ascend_commit: str
+    cann_version: str
+    torch_version: str
+    torch_npu_version: str
+
+
+@pytest.fixture
+def env_config() -> EnvConfig:
+    return EnvConfig(
+        vllm_version=os.getenv("VLLM_VERSION", "unknown"),
+        vllm_commit=os.getenv("VLLM_COMMIT", "unknown"),
+        vllm_ascend_version=os.getenv("VLLM_ASCEND_VERSION", "unknown"),
+        vllm_ascend_commit=os.getenv("VLLM_ASCEND_COMMIT", "unknown"),
+        cann_version=os.getenv("CANN_VERSION", "unknown"),
+        torch_version=os.getenv("TORCH_VERSION", "unknown"),
+        torch_npu_version=os.getenv("TORCH_NPU_VERSION", "unknown"),
+    )
+
+
+def build_serve_args(eval_config: dict) -> list[str]:
+    """Convert the serve: section of the YAML into a vllm serve CLI args list.
+
+    Example — serve: {tensor_parallel_size: 2, dtype: auto} becomes:
+        ["--tensor-parallel-size", "2", "--dtype", "auto"]
+    """
+    serve_cfg = eval_config.get("serve", {})
+    flag_map = {
+        "tensor_parallel_size": "--tensor-parallel-size",
+        "dtype": "--dtype",
+        "max_model_len": "--max-model-len",
+        "gpu_memory_utilization": "--gpu-memory-utilization",
+        "trust_remote_code": "--trust-remote-code",
+        "enforce_eager": "--enforce-eager",
+        "quantization": "--quantization",
+    }
+    args: list[str] = []
+    for key, flag in flag_map.items():
+        value = serve_cfg.get(key)
+        if value is None:
+            continue
+        if isinstance(value, bool):
+            if value:
+                args.append(flag)
+        else:
+            args.extend([flag, str(value)])
+    return args
+
+
+def audio_to_wav_bytes(audio_array: np.ndarray, sample_rate: int) -> bytes:
+    """Convert a numpy audio array to in-memory WAV bytes at the given sample rate."""
+    buf = io.BytesIO()
+    # Ensure int16 encoding for maximum API compatibility.
+    if audio_array.dtype != np.int16:
+        if np.issubdtype(audio_array.dtype, np.floating):
+            audio_array = np.clip(audio_array, -1.0, 1.0)
+            audio_array = (audio_array * 32767).astype(np.int16)
+        else:
+            audio_array = audio_array.astype(np.int16)
+    wav_io.write(buf, sample_rate, audio_array)
+    return buf.getvalue()
+
+
+def normalize_text(text: str) -> str:
+    """Normalize text for WER calculation: lowercase, strip punctuation, collapse whitespace."""
+    text = text.lower()
+    text = text.translate(_PUNCT_TABLE)
+    text = " ".join(text.split())
+    return text
+
+
+def transcribe_batch(client, model_name: str, audio_items: list[dict], language: str) -> list[str]:
+    """Call /v1/audio/transcriptions for a list of audio items.
+
+    Each item in audio_items must have keys: audio_array (np.ndarray), sample_rate (int).
+    Returns the raw transcription strings in the same order.
+    """
+    hypotheses: list[str] = []
+    for item in audio_items:
+        wav_bytes = audio_to_wav_bytes(item["audio_array"], item["sample_rate"])
+        response = client.audio.transcriptions.create(
+            model=model_name,
+            file=("audio.wav", wav_bytes, "audio/wav"),
+            language=language,
+        )
+        hypotheses.append(response.text)
+    return hypotheses
+
+
+def generate_asr_report(
+    eval_config: dict,
+    report_data: dict,
+    report_dir: str,
+    env_config: EnvConfig,
+) -> None:
+    """Write a Markdown accuracy report using the same Jinja2 template as lm_eval tests."""
+    env = Environment(loader=FileSystemLoader(TEST_DIR))
+    template = env.get_template("report_template.md")
+
+    serve_cfg = eval_config.get("serve", {})
+    tp_size = serve_cfg.get("tensor_parallel_size", 1)
+    ep_enabled = serve_cfg.get("enable_expert_parallel", False)
+    enforce_eager = serve_cfg.get("enforce_eager", False)
+
+    parallel_mode = f"TP{tp_size}"
+    if ep_enabled:
+        parallel_mode += " + EP"
+    execution_model = "Eager" if enforce_eager else "ACLGraph"
+
+    model_args_str = ",".join(f"{k}={v}" for k, v in serve_cfg.items())
+
+    report_content = template.render(
+        vllm_version=env_config.vllm_version,
+        vllm_commit=env_config.vllm_commit,
+        vllm_ascend_version=env_config.vllm_ascend_version,
+        vllm_ascend_commit=env_config.vllm_ascend_commit,
+        cann_version=env_config.cann_version,
+        torch_version=env_config.torch_version,
+        torch_npu_version=env_config.torch_npu_version,
+        hardware=eval_config.get("hardware", "unknown"),
+        model_name=eval_config["model_name"],
+        model_args=f"'{model_args_str}'",
+        model_type=eval_config.get("model_type", "vllm-asr"),
+        datasets=",".join(t["name"] for t in eval_config["tasks"]),
+        apply_chat_template=False,
+        fewshot_as_multiturn=False,
+        limit=eval_config.get("limit", "N/A"),
+        batch_size=eval_config.get("batch_size", 8),
+        num_fewshot="N/A",
+        rows=report_data["rows"],
+        parallel_mode=parallel_mode,
+        execution_model=execution_model,
+        show_command=False,
+    )
+
+    report_path = os.path.join(report_dir, f"{os.path.basename(eval_config['model_name'])}.md")
+    os.makedirs(os.path.dirname(report_path), exist_ok=True)
+    with open(report_path, "w", encoding="utf-8") as f:
+        f.write(report_content)
+
+
+def test_asr_eval_param(config_filename, tp_size, report_dir, env_config):
+    """Parametrised ASR accuracy test driven by a YAML config file.
+
+    Skips automatically when the config's model_type is not "vllm-asr".
+    """
+    eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
+
+    if eval_config.get("model_type", "vllm") != "vllm-asr":
+        pytest.skip(f"Skipping non-ASR config (model_type={eval_config.get('model_type', 'vllm')})")
+
+    model_name: str = eval_config["model_name"]
+    language: str = eval_config.get("language", "en")
+    limit: int | None = eval_config.get("limit", None)
+    batch_size: int = eval_config.get("batch_size", 8)
+
+    # Build serve args, letting --tp-size CLI flag override the YAML value.
+    serve_args = build_serve_args(eval_config)
+    if tp_size and tp_size != "1":
+        # Drop any --tensor-parallel-size already in serve_args, then append
+        # the CLI-supplied value so it takes precedence over the YAML setting.
+        it = iter(serve_args)
+        serve_args = [a for a in it if a != "--tensor-parallel-size" or not next(it, None)]
+        serve_args += ["--tensor-parallel-size", str(tp_size)]
+
+    print(f"\nStarting vllm serve for {model_name}")
+    print(f"  serve args: {serve_args}")
+
+    success = True
+    report_data: dict[str, list[dict]] = {"rows": []}
+
+    server_port = get_open_port()
+    serve_args = serve_args + ["--port", str(server_port)]
+    with RemoteOpenAIServer(model_name, serve_args, server_port=server_port, auto_port=False) as server:
+        client = server.get_client()
+
+        for task in eval_config["tasks"]:
+            task_name: str = task["name"]
+            dataset_name: str = task["dataset"]
+            split: str = task["split"]
+            dataset_config_name: str | None = task.get("dataset_config")
+            audio_col: str = task.get("audio_column", "audio")
+            text_col: str = task.get("text_column", "text")
+
+            split_expr = f"{split}[:{limit}]" if limit is not None else split
+            print(f"\nLoading dataset via modelscope: {dataset_name} / {dataset_config_name} ({split_expr})")
+            ds = MsDataset.load(
+                dataset_name,
+                subset_name=dataset_config_name,
+                split=split_expr,
+            )
+            if limit is not None:
+                ds = ds.select(range(min(limit, len(ds))))
+
+            # Disable automatic audio decoding so we can use soundfile instead
+            # of torchcodec (which requires CUDA libs unavailable on Ascend NPU).
+            if hasattr(ds, "cast_column"):
+                ds = ds.cast_column(audio_col, Audio(decode=False))
+
+            print(f"  {len(ds)} samples to evaluate")
+
+            # Collect audio items and references in batches.
+            all_hypotheses: list[str] = []
+            all_references: list[str] = []
+
+            for batch_start in range(0, len(ds), batch_size):
+                batch = ds.select(range(batch_start, min(batch_start + batch_size, len(ds))))
+                audio_items = []
+                for sample in batch:
+                    raw = sample[audio_col]
+                    if isinstance(raw, dict) and "bytes" in raw and raw["bytes"] is not None:
+                        audio_array, sample_rate = sf.read(io.BytesIO(raw["bytes"]))
+                    elif isinstance(raw, dict) and "path" in raw and raw["path"] is not None:
+                        audio_array, sample_rate = sf.read(raw["path"])
+                    else:
+                        # Already decoded (e.g. MsDataset with native decoding)
+                        audio_array = raw["array"]
+                        sample_rate = raw["sampling_rate"]
+                    audio_items.append({"audio_array": audio_array, "sample_rate": sample_rate})
+                references = [sample[text_col] for sample in batch]
+
+                hypotheses = transcribe_batch(client, model_name, audio_items, language)
+                all_hypotheses.extend(hypotheses)
+                all_references.extend(references)
+
+                if (batch_start // batch_size + 1) % 5 == 0:
+                    print(f"  processed {batch_start + len(batch)}/{len(ds)} samples …")
+
+            # Normalise both sides before WER calculation.
+            norm_hypotheses = [normalize_text(h) for h in all_hypotheses]
+            norm_references = [normalize_text(r) for r in all_references]
+
+            measured_wer = round(jiwer.wer(norm_references, norm_hypotheses), 4)
+            print(f"\n{task_name} WER = {measured_wer:.4f}")
+
+            for metric in task["metrics"]:
+                if metric["name"] != "wer":
+                    continue
+                ground_truth = metric["value"]
+                # Pass if measured WER is at or below the threshold (better is OK);
+                # allow up to RTOL relative degradation above the threshold.
+                task_success = measured_wer <= ground_truth * (1 + RTOL)
+                success = success and task_success
+
+                status = "✅" if task_success else "❌"
+                print(f"{task_name} | wer: ground_truth={ground_truth} | measured={measured_wer} | {status}")
+
+                report_data["rows"].append(
+                    {
+                        "task": task_name,
+                        "metric": "wer",
+                        "value": f"{status}{measured_wer}",
+                        "stderr": "N/A",
+                    }
+                )
+
+    generate_asr_report(eval_config, report_data, report_dir, env_config)
+    assert success, "One or more ASR tasks exceeded the WER tolerance. See output above."

tests/e2e/models/test_lm_eval_correctness.py

@@ -7,7 +7,7 @@ import pytest
 import yaml
 from jinja2 import Environment, FileSystemLoader
 
-RTOL = 0.03
+RTOL = 0.05
 TEST_DIR = os.path.dirname(__file__)
 
 
@@ -24,33 +24,39 @@ class EnvConfig:
 
 @pytest.fixture
 def env_config() -> EnvConfig:
-    return EnvConfig(vllm_version=os.getenv('VLLM_VERSION', 'unknown'),
-                     vllm_commit=os.getenv('VLLM_COMMIT', 'unknown'),
-                     vllm_ascend_version=os.getenv('VLLM_ASCEND_VERSION',
-                                                   'unknown'),
-                     vllm_ascend_commit=os.getenv('VLLM_ASCEND_COMMIT',
-                                                  'unknown'),
-                     cann_version=os.getenv('CANN_VERSION', 'unknown'),
-                     torch_version=os.getenv('TORCH_VERSION', 'unknown'),
-                     torch_npu_version=os.getenv('TORCH_NPU_VERSION',
-                                                 'unknown'))
+    return EnvConfig(
+        vllm_version=os.getenv("VLLM_VERSION", "unknown"),
+        vllm_commit=os.getenv("VLLM_COMMIT", "unknown"),
+        vllm_ascend_version=os.getenv("VLLM_ASCEND_VERSION", "unknown"),
+        vllm_ascend_commit=os.getenv("VLLM_ASCEND_COMMIT", "unknown"),
+        cann_version=os.getenv("CANN_VERSION", "unknown"),
+        torch_version=os.getenv("TORCH_VERSION", "unknown"),
+        torch_npu_version=os.getenv("TORCH_NPU_VERSION", "unknown"),
+    )
 
 
 def build_model_args(eval_config, tp_size):
-    trust_remote_code = eval_config.get("trust_remote_code", False)
-    max_model_len = eval_config.get("max_model_len", 4096)
+    serve_cfg = eval_config.get("serve", {})
+    trust_remote_code = serve_cfg.get("trust_remote_code", False)
+    max_model_len = serve_cfg.get("max_model_len", 4096)
+    dtype = serve_cfg.get("dtype", "auto")
     model_args = {
         "pretrained": eval_config["model_name"],
         "tensor_parallel_size": tp_size,
-        "dtype": "auto",
+        "dtype": dtype,
         "trust_remote_code": trust_remote_code,
         "max_model_len": max_model_len,
     }
     for s in [
-            "max_images", "gpu_memory_utilization", "enable_expert_parallel",
-            "tensor_parallel_size", "enforce_eager"
+        "max_images",
+        "gpu_memory_utilization",
+        "enable_expert_parallel",
+        "tensor_parallel_size",
+        "enforce_eager",
+        "enable_thinking",
+        "quantization",
     ]:
-        val = eval_config.get(s, None)
+        val = serve_cfg.get(s, None)
         if val is not None:
             model_args[s] = val
 
@@ -66,7 +72,7 @@ def generate_report(tp_size, eval_config, report_data, report_dir, env_config):
     model_args = build_model_args(eval_config, tp_size)
 
     parallel_mode = f"TP{model_args.get('tensor_parallel_size', 1)}"
-    if model_args.get('enable_expert_parallel', False):
+    if model_args.get("enable_expert_parallel", False):
         parallel_mode += " + EP"
 
     execution_model = f"{'Eager' if model_args.get('enforce_eager', False) else 'ACLGraph'}"
@@ -82,7 +88,7 @@ def generate_report(tp_size, eval_config, report_data, report_dir, env_config):
         hardware=eval_config.get("hardware", "unknown"),
         model_name=eval_config["model_name"],
         model_args=f"'{','.join(f'{k}={v}' for k, v in model_args.items())}'",
-        model_type=eval_config.get("model", "vllm"),
+        model_type=eval_config.get("model_type", "vllm"),
         datasets=",".join([task["name"] for task in eval_config["tasks"]]),
         apply_chat_template=eval_config.get("apply_chat_template", True),
         fewshot_as_multiturn=eval_config.get("fewshot_as_multiturn", True),
@@ -91,24 +97,27 @@ def generate_report(tp_size, eval_config, report_data, report_dir, env_config):
         num_fewshot=eval_config.get("num_fewshot", "N/A"),
         rows=report_data["rows"],
         parallel_mode=parallel_mode,
-        execution_model=execution_model)
+        execution_model=execution_model,
+    )
 
-    report_output = os.path.join(
-        report_dir, f"{os.path.basename(eval_config['model_name'])}.md")
+    report_output = os.path.join(report_dir, f"{os.path.basename(eval_config['model_name'])}.md")
     os.makedirs(os.path.dirname(report_output), exist_ok=True)
-    with open(report_output, 'w', encoding='utf-8') as f:
+    with open(report_output, "w", encoding="utf-8") as f:
         f.write(report_content)
 
 
-def test_lm_eval_correctness_param(config_filename, tp_size, report_dir,
-                                   env_config):
+def test_lm_eval_correctness_param(config_filename, tp_size, report_dir, env_config):
     eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
+
+    if eval_config.get("model_type", "vllm") == "vllm-asr":
+        pytest.skip("Skipping ASR config, use test_asr_eval.py instead")
+
     model_args = build_model_args(eval_config, tp_size)
     success = True
     report_data: dict[str, list[dict]] = {"rows": []}
 
     eval_params = {
-        "model": eval_config.get("model", "vllm"),
+        "model": eval_config.get("model_type", "vllm"),
         "model_args": model_args,
         "tasks": [task["name"] for task in eval_config["tasks"]],
         "apply_chat_template": eval_config.get("apply_chat_template", True),
@@ -133,25 +142,26 @@ def test_lm_eval_correctness_param(config_filename, tp_size, report_dir,
             metric_name = metric["name"]
             ground_truth = metric["value"]
             measured_value = round(task_result[metric_name], 4)
-            task_success = bool(
-                np.isclose(ground_truth, measured_value, rtol=RTOL))
+            task_success = bool(np.isclose(ground_truth, measured_value, rtol=RTOL))
             success = success and task_success
 
-            print(f"{task_name} | {metric_name}: "
-                  f"ground_truth={ground_truth} | measured={measured_value} | "
-                  f"success={'✅' if task_success else '❌'}")
+            print(
+                f"{task_name} | {metric_name}: "
+                f"ground_truth={ground_truth} | measured={measured_value} | "
+                f"success={'✅' if task_success else '❌'}"
+            )
 
-            report_data["rows"].append({
-                "task":
-                task_name,
-                "metric":
-                metric_name,
-                "value":
-                f"✅{measured_value}" if success else f"❌{measured_value}",
-                "stderr":
-                task_result[
-                    metric_name.replace(',', '_stderr,') if metric_name ==
-                    "acc,none" else metric_name.replace(',', '_stderr,')]
-            })
+            report_data["rows"].append(
+                {
+                    "task": task_name,
+                    "metric": metric_name,
+                    "value": f"✅{measured_value}" if success else f"❌{measured_value}",
+                    "stderr": task_result[
+                        metric_name.replace(",", "_stderr,")
+                        if metric_name == "acc,none"
+                        else metric_name.replace(",", "_stderr,")
+                    ],
+                }
+            )
     generate_report(tp_size, eval_config, report_data, report_dir, env_config)
     assert success

tests/e2e/models/test_rm_eval_correctness.py

@@ -0,0 +1,255 @@
+import os
+from dataclasses import dataclass
+
+import pytest
+import regex as re
+import yaml
+from jinja2 import Environment, FileSystemLoader
+from modelscope.msdatasets import MsDataset  # type: ignore[import-untyped]
+
+from tests.e2e.conftest import VllmRunner
+
+# Allow up to 5 % relative degradation from the declared ground-truth accuracy.
+RTOL = 0.05
+
+TEST_DIR = os.path.dirname(__file__)
+
+# Default system prompt for Qwen2.5-Math-RM style models.
+_DEFAULT_SYSTEM_PROMPT = "Please reason step by step, and put your final answer within \\boxed{}."
+
+
+@dataclass
+class EnvConfig:
+    vllm_version: str
+    vllm_commit: str
+    vllm_ascend_version: str
+    vllm_ascend_commit: str
+    cann_version: str
+    torch_version: str
+    torch_npu_version: str
+
+
+@pytest.fixture
+def env_config() -> EnvConfig:
+    return EnvConfig(
+        vllm_version=os.getenv("VLLM_VERSION", "unknown"),
+        vllm_commit=os.getenv("VLLM_COMMIT", "unknown"),
+        vllm_ascend_version=os.getenv("VLLM_ASCEND_VERSION", "unknown"),
+        vllm_ascend_commit=os.getenv("VLLM_ASCEND_COMMIT", "unknown"),
+        cann_version=os.getenv("CANN_VERSION", "unknown"),
+        torch_version=os.getenv("TORCH_VERSION", "unknown"),
+        torch_npu_version=os.getenv("TORCH_NPU_VERSION", "unknown"),
+    )
+
+
+def format_rm_input(system_prompt: str, problem: str, solution: str) -> str:
+    """Format a (problem, solution) pair using the Qwen chat template."""
+    return (
+        f"<|im_start|>system\n{system_prompt}<|im_end|>\n"
+        f"<|im_start|>user\n{problem}<|im_end|>\n"
+        f"<|im_start|>assistant\n{solution}<|im_end|>"
+    )
+
+
+def perturb_answer(solution: str) -> str:
+    """Create an obviously wrong solution for a GSM8K-style answer string.
+
+    GSM8K answers end with ``#### <number>``.  We replace that number with
+    ``correct * 3 + 137`` so the final answer is clearly incorrect while the
+    reasoning chain looks plausible.
+    """
+    match = re.search(r"####\s*([\d,]+(?:\.\d+)?)", solution)
+    if match:
+        num_str = match.group(1).replace(",", "")
+        try:
+            correct_num = float(num_str)
+            wrong_num = int(correct_num * 3 + 137)
+            return solution[: match.start()] + f"#### {wrong_num}"
+        except ValueError:
+            pass
+    # Fallback: append an unmistakably wrong sentinel answer.
+    return solution + "\n#### -999999"
+
+
+def extract_reward_score(reward_output) -> float:
+    """Extract a scalar score from VllmRunner.reward() output for one sample.
+
+    VllmRunner.reward() returns list[list[float]] or list[Tensor]; for a reward
+    model with a single output the inner list has one element.  For a token-level
+    reward model the output is a 2-D tensor [seq_len, 1]; in both cases we take
+    the last element (final-step score).
+    """
+    if isinstance(reward_output, (list, tuple)):
+        return float(reward_output[-1])
+    # Tensor (e.g. shape [seq_len, 1] from a token-level reward model)
+    return float(reward_output.flatten()[-1].item())
+
+
+def generate_rm_report(
+    eval_config: dict,
+    report_data: dict,
+    report_dir: str,
+    env_config: EnvConfig,
+) -> None:
+    """Write a Markdown accuracy report using the shared Jinja2 template."""
+    jinja_env = Environment(loader=FileSystemLoader(TEST_DIR))
+    template = jinja_env.get_template("report_template.md")
+
+    serve_cfg = eval_config.get("serve", {})
+    tp_size = serve_cfg.get("tensor_parallel_size", 1)
+    ep_enabled = serve_cfg.get("enable_expert_parallel", False)
+    enforce_eager = serve_cfg.get("enforce_eager", False)
+
+    parallel_mode = f"TP{tp_size}"
+    if ep_enabled:
+        parallel_mode += " + EP"
+    execution_model = "Eager" if enforce_eager else "ACLGraph"
+    model_args_str = ",".join(f"{k}={v}" for k, v in serve_cfg.items())
+
+    report_content = template.render(
+        vllm_version=env_config.vllm_version,
+        vllm_commit=env_config.vllm_commit,
+        vllm_ascend_version=env_config.vllm_ascend_version,
+        vllm_ascend_commit=env_config.vllm_ascend_commit,
+        cann_version=env_config.cann_version,
+        torch_version=env_config.torch_version,
+        torch_npu_version=env_config.torch_npu_version,
+        hardware=eval_config.get("hardware", "unknown"),
+        model_name=eval_config["model_name"],
+        model_args=f"'{model_args_str}'",
+        model_type=eval_config.get("model_type", "vllm-rm"),
+        datasets=",".join(t["name"] for t in eval_config["tasks"]),
+        apply_chat_template=False,
+        fewshot_as_multiturn=False,
+        limit=eval_config.get("limit", "N/A"),
+        batch_size=eval_config.get("batch_size", 4),
+        num_fewshot="N/A",
+        rows=report_data["rows"],
+        parallel_mode=parallel_mode,
+        execution_model=execution_model,
+    )
+
+    report_path = os.path.join(report_dir, f"{os.path.basename(eval_config['model_name'])}.md")
+    os.makedirs(os.path.dirname(report_path), exist_ok=True)
+    with open(report_path, "w", encoding="utf-8") as f:
+        f.write(report_content)
+
+
+def test_rm_eval_param(config_filename, tp_size, report_dir, env_config):
+    """Parametrised reward-model accuracy test driven by a YAML config file.
+
+    Skips automatically when the config's model_type is not "vllm-rm".
+    """
+    eval_config = yaml.safe_load(config_filename.read_text(encoding="utf-8"))
+
+    if eval_config.get("model_type", "vllm") != "vllm-rm":
+        pytest.skip(f"Skipping non-RM config (model_type={eval_config.get('model_type', 'vllm')})")
+
+    model_name: str = eval_config["model_name"]
+    limit: int | None = eval_config.get("limit", None)
+    batch_size: int = eval_config.get("batch_size", 4)
+    system_prompt: str = eval_config.get("system_prompt", _DEFAULT_SYSTEM_PROMPT)
+    serve_cfg: dict = eval_config.get("serve", {})
+
+    # CLI --tp-size takes precedence over the YAML tensor_parallel_size.
+    effective_tp = int(tp_size) if (tp_size and tp_size != "1") else int(serve_cfg.get("tensor_parallel_size", 1))
+
+    runner_kwargs: dict = {
+        k: v
+        for k, v in {
+            "runner": "pooling",
+            "dtype": serve_cfg.get("dtype", "auto"),
+            "tensor_parallel_size": effective_tp,
+            "enforce_eager": serve_cfg.get("enforce_eager", False),
+            "max_model_len": serve_cfg.get("max_model_len"),
+            "gpu_memory_utilization": serve_cfg.get("gpu_memory_utilization"),
+        }.items()
+        if v is not None
+    }
+
+    print(f"\nLoading reward model: {model_name}")
+    print(f"  VllmRunner kwargs: {runner_kwargs}")
+
+    success = True
+    report_data: dict[str, list[dict]] = {"rows": []}
+
+    with VllmRunner(model_name, **runner_kwargs) as vllm_model:
+        for task in eval_config["tasks"]:
+            task_name: str = task["name"]
+            dataset_name: str = task["dataset"]
+            split: str = task["split"]
+            dataset_config_name: str | None = task.get("dataset_config")
+            task_type: str = task.get("task_type", "correctness")
+
+            # Column names for "correctness" tasks (e.g. GSM8K).
+            problem_col: str = task.get("problem_column", "question")
+            solution_col: str = task.get("solution_column", "answer")
+
+            # Column names for "pairwise" tasks (e.g. reward-bench).
+            prompt_col: str = task.get("prompt_column", "prompt")
+            chosen_col: str = task.get("chosen_column", "chosen")
+            rejected_col: str = task.get("rejected_column", "rejected")
+
+            split_expr = f"{split}[:{limit}]" if limit is not None else split
+            print(f"\nLoading dataset via ModelScope: {dataset_name} / {dataset_config_name} ({split_expr})")
+
+            # MsDataset may bypass the HF_HUB_OFFLINE lock; patch temporarily.
+            ds = MsDataset.load(
+                dataset_name,
+                subset_name=dataset_config_name,
+                split=split_expr,
+            )
+            print(f"  {len(ds)} samples to evaluate (task_type={task_type})")
+
+            correct_count = 0
+            total_count = 0
+
+            for batch_start in range(0, len(ds), batch_size):
+                batch = ds.select(range(batch_start, min(batch_start + batch_size, len(ds))))
+
+                if task_type == "pairwise":
+                    positive_texts = [format_rm_input(system_prompt, s[prompt_col], s[chosen_col]) for s in batch]
+                    negative_texts = [format_rm_input(system_prompt, s[prompt_col], s[rejected_col]) for s in batch]
+                else:
+                    positive_texts = [format_rm_input(system_prompt, s[problem_col], s[solution_col]) for s in batch]
+                    negative_texts = [
+                        format_rm_input(system_prompt, s[problem_col], perturb_answer(s[solution_col])) for s in batch
+                    ]
+
+                pos_rewards = vllm_model.reward(positive_texts)
+                neg_rewards = vllm_model.reward(negative_texts)
+
+                for pos_r, neg_r in zip(pos_rewards, neg_rewards):
+                    if extract_reward_score(pos_r) > extract_reward_score(neg_r):
+                        correct_count += 1
+                    total_count += 1
+
+                if (batch_start // batch_size + 1) % 5 == 0:
+                    print(f"  processed {batch_start + len(batch)}/{len(ds)} samples …")
+
+            measured_accuracy = round(correct_count / total_count, 4) if total_count > 0 else 0.0
+            print(f"\n{task_name} accuracy = {measured_accuracy:.4f}")
+
+            for metric in task["metrics"]:
+                if metric["name"] != "accuracy":
+                    continue
+                ground_truth = metric["value"]
+                # Pass if measured accuracy meets or exceeds the threshold
+                # (allow up to RTOL relative degradation).
+                task_success = measured_accuracy >= ground_truth * (1 - RTOL)
+                success = success and task_success
+
+                status = "✅" if task_success else "❌"
+                print(f"{task_name} | accuracy: ground_truth={ground_truth} | measured={measured_accuracy} | {status}")
+
+                report_data["rows"].append(
+                    {
+                        "task": task_name,
+                        "metric": "accuracy",
+                        "value": f"{status}{measured_accuracy}",
+                        "stderr": "N/A",
+                    }
+                )
+
+    generate_rm_report(eval_config, report_data, report_dir, env_config)
+    assert success, "One or more RM tasks did not meet the accuracy threshold. See output above."