taskType: asr

framework: funasr

configParams:
framework: funasr
lang: zh
ref_config:
  gpu_num: 1
  values:
    command: [python3, main.py, --port, '80', --model_dir, /model, --use_gpu, --model_type,
      sensevoice]
sut_config:
  gpu_num: 1
  values:
    command: [python3, main.py, --port, '80', --model_dir, /model, --use_gpu, --model_type,
      sensevoice]


configParams:
docker_image: harbor-contest.4pd.io/sunjichen/funasr:ilu-mr100-service-0.2
nv_docker_image: harbor-contest.4pd.io/sunjichen/funasr:a100-server-0.2
framework: funasr
lang: zh
sut_config:
  gpu_num: 1
  values:
    command: [python3, main.py, --port, '8000', --model_dir, /model, --model_type,
      paraformer]
ref_config:
  gpu_num: 1
  values:
    command: [python3, main.py, --port, '8000', --model_dir, /model, --model_type,
      paraformer]


framework: sherpa-onnx

configParams:
lang: zh
framework: sherpa-onnx
sut_config:
  gpu_num: 1
  values:
    command:
    - python3
    - main_sherpa.py
    - --port
    - '80'
    - --model_dir
    - /model
    - --use_gpu
    - --model_type
    - dolphin_ctc
    - --offline_model
ref_config:
  gpu_num: 1
  values:
    command:
    - python3
    - main_sherpa.py
    - --port
    - '80'
    - --model_dir
    - /model
    - --use_gpu
    - --model_type
    - dolphin_ctc
    - --offline_model


framework: transformers

configParams:
lang: zh
framework: transformers
sut_config:
  gpu_num: 1
  values:
    command:
    - python3
    - main_transformers.py
    - --port
    - '80'
    - --model_dir
    - /model
    - --use_gpu
ref_config:
  gpu_num: 1
  values:
    command:
    - python3
    - main_transformers.py
    - --port
    - '80'
    - --model_dir
    - /model
    - --use_gpu


framework: sentence-transformers

configParams:
framework: sentence-transformers
sut_config:
  gpu_num: 1
  values:
    command:
    - /opt/conda/bin/python
    - -m
    - uvicorn
    - server:app
    - --host
    - 0.0.0.0
    - --port
    - '8000'
ref_config:
  gpu_num: 1
  values:
    command:
    - python3
    - -m
    - uvicorn
    - server:app
    - --host
    - 0.0.0.0
    - --port
    - '8000'


configParams:
framework: transformers
sut_config:
  gpu_num: 1
  values:
    env:
    - name: CONFIG_JSON
      value: |
        {
          "torch_dtype": "auto",
          "handle_impossible_answer": false,
          "score_threshold": 0.0,
          "max_answer_len": 30,
          "max_seq_len": 512,
          "doc_stride": 128
        }
    command:
    - python3
    - main_qa.py
    - --model_dir
    - /model
    - --port
    - '80'
ref_config:
  gpu_num: 1
  values:
    env:
    - name: CONFIG_JSON
      value: |
        {
          "torch_dtype": "auto",
          "handle_impossible_answer": false,
          "score_threshold": 0.0,
          "max_answer_len": 30,
          "max_seq_len": 512,
          "doc_stride": 128
        }
    command:
    - python3
    - main_qa.py
    - --model_dir
    - /model
    - --port
    - '80'


framework: llamacpp

configParams:
framework: llamacpp
api: completion
lang: en
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.1
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command:
    - /app/llama-server
    - --model
    - /model/Math-Code-Llama3.1-8B.i1-Q4_0.gguf
    - --alias
    - llm
    - --threads
    - '20'
    - --n-gpu-layers
    - '128'
    - --ctx-size
    - '4096'
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --jinja
    - --flash-attn
    - 'off'
    - --no-mmap
    - --sync-to-temp
ref_config:
  gpu_num: 1
  values:
    command:
    - /workspace/llama.cpp/build/bin/llama-server
    - --model
    - /model/Math-Code-Llama3.1-8B.i1-Q4_0.gguf
    - --alias
    - llm
    - --threads
    - '20'
    - --n-gpu-layers
    - '128'
    - --ctx-size
    - '4096'
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --jinja
    - --flash-attn
    - 'off'


configParams:
docker_image: PLACEHOLDER_DOCKER_IMAGE
nv_docker_image: PLACEHOLDER_DOCKER_IMAGE
api: completion
framework: transformers
nv_framework: vllm
lang: en
max_model_len: 4096
sut_config:
  gpu_num: 1
  values:
    env:
    - name: CONFIG_JSON
      value: |
        {
          "model_class": "AutoModelForCausalLM",
          "torch_dtype": "auto",
          "model_kwargs": {
            "trust_remote_code": true,
            "device_map": "auto"
          },
          "tokenizer_kwargs": {
            "use_fast": true,
            "trust_remote_code": true
          },
          "generate_kwargs": {
            "temperature": 0.01,
            "top_p": 0.8,
            "repetition_penalty": 1.0,
            "max_new_tokens": 1024
          }
        }
    command:
    - python3
    - transformers_server.py
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '80'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - --enforce-eager
    - --trust-remote-code
    - -tp
    - '1'


framework: vllm

configParams:
docker_image: git.modelhub.org.cn:9443/enginex-iluvatar-bi150/vllm:0.8.3
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
api: completion
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.2
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '80'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - --gpu-memory-utilization
    - '0.9'
    - --enforce-eager
    - --trust-remote-code
    - -tp
    - '1'
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '80'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - --enforce-eager
    - --trust-remote-code
    - -tp
    - '1'


taskType: text-generation

configParams:
docker_image: PLACEHOLDER_DOCKER_IMAGE
nv_docker_image: PLACEHOLDER_DOCKER_IMAGE
framework: llamacpp
api: completion
max_tokens: 1024
max_model_len: 2048
sut_config:
  gpu_num: 1
  values:
    command:
    - /workspace/llama.cpp/build_ascend/bin/llama-server
    - --model
    - /model/Free_Sydney_V2_Mistral_7b.Q8_0.gguf
    - --alias
    - llm
    - --threads
    - '20'
    - --n-gpu-layers
    - '128'
    - --ctx-size
    - '2048'
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --jinja
    - --flash-attn
    - 'off'
ref_config:
  gpu_num: 1
  values:
    command:
    - /workspace/llama.cpp/build/bin/llama-server
    - --model
    - /model/Free_Sydney_V2_Mistral_7b.Q8_0.gguf
    - --alias
    - llm
    - --threads
    - '20'
    - --n-gpu-layers
    - '128'
    - --ctx-size
    - '2048'
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --jinja
    - --flash-attn
    - 'off'


configParams:
framework: llamacpp
api: chat
lang: zh
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.1
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command: [/workspace/llama.cpp/build_ascend/bin/llama-server, --model, /model/Gemma2-FT-ai-medical-chatbot.Q8_0.gguf,
      --alias, llm, --threads, '16', --n-gpu-layers, '128', --prio, '3', --min_p,
      '0.01', --ctx-size, '4096', --host, 0.0.0.0, --port, '3316', --jinja, --flash-attn,
      'off']
ref_config:
  gpu_num: 1
  values:
    command: [/workspace/llama.cpp/build/bin/llama-server, --model, /model/Gemma2-FT-ai-medical-chatbot.Q8_0.gguf,
      --alias, llm, --threads, '16', --n-gpu-layers, '128', --prio, '3', --min_p,
      '0.01', --ctx-size, '4096', --host, 0.0.0.0, --port, '80', --jinja]


configParams:
framework: llamacpp
api: completion
lang: en
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.1
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command: [/app/llama-server, --model, /model/Math-Code-Llama3.1-8B.i1-Q4_0.gguf,
      --alias, llm, --threads, '20', --n-gpu-layers, '128', --ctx-size, '4096', --host,
      0.0.0.0, --port, '8000', --jinja, --flash-attn, 'off', --no-mmap, --sync-to-temp]
ref_config:
  gpu_num: 1
  values:
    command: [/workspace/llama.cpp/build/bin/llama-server, --model, /model/Math-Code-Llama3.1-8B.i1-Q4_0.gguf,
      --alias, llm, --threads, '20', --n-gpu-layers, '128', --ctx-size, '4096', --host,
      0.0.0.0, --port, '8000', --jinja, --flash-attn, 'off']


configParams:
docker_image: PLACEHOLDER_DOCKER_IMAGE
nv_docker_image: PLACEHOLDER_DOCKER_IMAGE
framework: llamacpp
api: completion
max_tokens: 1024
max_model_len: 2048
sut_config:
  gpu_num: 1
  values:
    command:
    - /app/llama-server
    - --model
    - /model/MediQwen-4B.Q8_0.gguf
    - --alias
    - llm
    - --threads
    - '20'
    - --n-gpu-layers
    - '128'
    - --ctx-size
    - '2048'
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --jinja
    - --flash-attn
    - 'off'
ref_config:
  gpu_num: 1
  values:
    command:
    - /workspace/llama.cpp/build/bin/llama-server
    - --model
    - /model/MediQwen-4B.Q8_0.gguf
    - --alias
    - llm
    - --threads
    - '20'
    - --n-gpu-layers
    - '128'
    - --ctx-size
    - '2048'
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --jinja
    - --flash-attn
    - 'off'


configParams:
framework: llamacpp
docker_image: PLACEHOLDER_DOCKER_IMAGE
nv_docker_image: PLACEHOLDER_NV_DOCKER_IMAGE
api: completion
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.2
top_p: 0.9
lang: en
max_model_len: 4096
sut_config:
  gpu_num: 1
  values:
    command: [/app/llama-server, --model, /model/qwen-qwen3-embedding-0.6b-f16.gguf,
      --alias, llm, --threads, '20', --n-gpu-layers, '999', --prio, '3', --min_p,
      '0.01', --ctx-size, '2048', --host, 0.0.0.0, --port, '8000', --jinja, --flash-attn,
      'off']
ref_config:
  gpu_num: 1
  values:
    command: [/workspace/llama.cpp/build/bin/llama-server, --model, /model/qwen-qwen3-embedding-0.6b-f16.gguf,
      --alias, llm, --threads, '20', --n-gpu-layers, '128', --ctx-size, '4096', --host,
      0.0.0.0, --port, '8000', --jinja]


framework: sglang

configParams:
docker_image: git.modelhub.org.cn:9443/enginex-sunrise/sglang:0.4.6.post2
nv_docker_image: harbor.4pd.io/hardcore-tech/public/sglang:v0.5.8
framework: sglang
api: completion
lang: en
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.1
top_p: 0.9
sut_config:
  values:
    command: [python3.10, -m, sglang.launch_server, --model-path, /model, --port,
      '17097', --served-model-name, llm, --context-length, '4096', --trust-remote-code,
      --host, 0.0.0.0, --disable-cuda-graph, --disable-radix-cache]
ref_config:
  gpu_num: 1
  values:
    command: [python3, -m, sglang.launch_server, --model-path, /model, --port, '8000',
      --served-model-name, llm, --context-length, '4096', --trust-remote-code, --host,
      0.0.0.0]


configParams:
docker_image: harbor-contest.4pd.io/sunjichen/funasr:ilu-mr100-service-0.2
nv_docker_image: harbor-contest.4pd.io/sunjichen/funasr:a100-server-0.2
api: completion
framework: transformers
nv_framework: vllm
lang: en
max_model_len: 4096
sut_config:
  gpu_num: 1
  values:
    env:
    - name: CONFIG_JSON
      value: |
        {
          "model_class": "AutoModelForCausalLM",
          "torch_dtype": "auto",
          "model_kwargs": {
            "trust_remote_code": true,
            "device_map": "auto"
          },
          "tokenizer_kwargs": {
            "use_fast": true,
            "trust_remote_code": true
          },
          "generate_kwargs": {
            "temperature": 0.01,
            "top_p": 0.8,
            "repetition_penalty": 1.0,
            "max_new_tokens": 1024
          }
        }
    command: [python3, transformers_server.py]
ref_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len,
      '4096', --enforce-eager, --trust-remote-code, -tp, '1']


configParams:
docker_image: PLACEHOLDER_DOCKER_IMAGE
nv_docker_image: PLACEHOLDER_DOCKER_IMAGE
framework: vllm
api: completion
lang: en
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.1
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '20644'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - -tp
    - '1'
    - --enforce-eager
    - --trust-remote-code
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '80'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - -tp
    - '1'
    - --enforce-eager
    - --trust-remote-code


configParams:
framework: vllm
docker_image: PLACEHOLDER_DOCKER_IMAGE
nv_docker_image: PLACEHOLDER_NV_DOCKER_IMAGE
api: completion
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.2
top_p: 0.9
lang: zh
max_model_len: 2048
sut_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '8000'
    - --served-model-name
    - llm
    - --max-model-len
    - '2048'
    - --dtype
    - auto
    - --gpu-memory-utilization
    - '0.95'
    - -tp
    - '1'
    - --enforce-eager
    - --trust-remote-code
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '8000'
    - --served-model-name
    - llm
    - --max-model-len
    - '2048'
    - --dtype
    - auto
    - --gpu-memory-utilization
    - '0.95'
    - -tp
    - '1'
    - --enforce-eager
    - --trust-remote-code


configParams:
framework: vllm
docker_image: git.modelhub.org.cn:9443/enginex/xc-llm-biren166m:26.01
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
api: completion
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.2
top_p: 0.9
lang: zh
max_model_len: 4096
sut_config:
  gpu_num: 4
  values:
    command: [/bin/bash, -ic, vllm serve /model --port 8000 --served-model-name llm
        --max-model-len 4096 --dtype auto --gpu-memory-utilization 0.9 --enforce-eager
        --trust-remote-code -tp 4 --host 0.0.0.0]
ref_config:
  gpu_num: 4
  values:
    command: [vllm, serve, /model, --port, '8000', --served-model-name, llm, --max-model-len,
      '4096', --dtype, auto, --gpu-memory-utilization, '0.95', -tp, '4', --enforce-eager,
      --trust-remote-code]


configParams:
docker_image: harbor.4pd.io/hardcore-tech/cambricon-mlu370-pytorch:v25.01-torch2.5.0-torchmlu1.24.1-ubuntu22.04-py310
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
sut_config:
  values:
    gpu_num: 1
    env:
    - {name: MAX_MODEL_LEN, value: 4096}
    command: [vllm, serve, /model, --port, '8000', --served-model-name, llm, --max-model-len,
      '4096', --trust-remote-code, --dtype, float16]
ref_config:
  values:
    cpu_num: 2
    gpu_num: 1
    env:
    - {name: MAX_MODEL_LEN, value: 4096}
    command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len,
      '4096', --trust-remote-code, --dtype, float16]


configParams:
framework: vllm
lang: en
max_model_len: 4096
sut_config:
  gpu_num: 1
  values:
    command: [/workspace/launch_service, --port, '80']
ref_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len,
      '4096', --enforce-eager, --trust-remote-code, -tp, '1']


configParams:
docker_image: git.modelhub.org.cn:9443/enginex-iluvatar-bi150/vllm:0.8.3
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
api: completion
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.2
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len,
      '4096', --gpu-memory-utilization, '0.9', --enforce-eager, --trust-remote-code,
      -tp, '1']
ref_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len,
      '4096', --enforce-eager, --trust-remote-code, -tp, '1']


configParams:
docker_image: harbor.4pd.io/hardcore-tech/iluvatar/llm-infer-iluvatar-mr:v0
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
max_model_len: 4096
lang: en
temperature: 0.4
repetition_penalty: 1.1
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '20644', --served-model-name, llm, --max-model-len,
      '4096', --gpu-memory-utilization, '0.9', --enforce-eager, --trust-remote-code,
      -tp, '1']
ref_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len,
      '4096', --enforce-eager, --trust-remote-code, -tp, '1']


configParams:
framework: vllm
docker_image: PLACEHOLDER_DOCKER_IMAGE
nv_docker_image: PLACEHOLDER_NV_DOCKER_IMAGE
api: completion
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.2
top_p: 0.9
lang: zh
max_model_len: 2048
sut_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '8000', --served-model-name, llm, --max-model-len,
      '2048', --dtype, auto, --gpu-memory-utilization, '0.95', -tp, '1', --enforce-eager,
      --trust-remote-code]
ref_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '8000', --served-model-name, llm, --max-model-len,
      '2048', --dtype, auto, --gpu-memory-utilization, '0.95', -tp, '1', --enforce-eager,
      --trust-remote-code]


configParams:
docker_image: harbor.4pd.io/mic-llm-x/slx-infer-kunlunxin:release-0.1-pipe-6-commit-f2eb860f
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
max_model_len: 4096
api: completion
sut_config:
  gpu_num: 1
  values:
    command: [sh, -c, rm -rf /workspace/xtrt_engine/fp16-1xpu && mkdir -p /workspace/xtrt_engine/fp16-1xpu
        && python3 examples/qwen/build.py --hf_model_dir /model --output_dir /workspace/xtrt_engine/fp16-1xpu
        --dtype float16 --use_gpt_attention_plugin float16 --world_size 1 --tp_size
        1 --remove_input_padding --opt_memory_use --paged_kv_cache --tokens_per_block
        64 --version 1.5 && exec python3 -m xtrt_llm.vllm.entrypoints.openai.api_server
        --model /model --engine_dir /workspace/xtrt_engine/fp16-1xpu --port 8000 --served-model-name
        llm --max-model-len 4096 --trust-remote-code --dtype float16]
ref_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len,
      '4096', --enforce-eager, --trust-remote-code, -tp, '1']


configParams:
docker_image: git.modelhub.org.cn:9443/enginex-metax/vllm:0.9.1
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
api: completion
lang: en
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.1
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command: [/opt/conda/bin/vllm, serve, /model, --port, '20644', --served-model-name,
      llm, --max-model-len, '4096', --gpu-memory-utilization, '0.9', -tp, '1', --enforce-eager,
      --trust-remote-code]
ref_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len,
      '4096', -tp, '1', --enforce-eager, --trust-remote-code]


configParams:
docker_image: PLACEHOLDER_DOCKER_IMAGE
nv_docker_image: PLACEHOLDER_DOCKER_IMAGE
framework: vllm
api: completion
lang: en
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.1
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '8000'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - --dtype
    - auto
    - --gpu-memory-utilization
    - '0.95'
    - -tp
    - '1'
    - --enforce-eager
    - --trust-remote-code
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '8000'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - --dtype
    - auto
    - --gpu-memory-utilization
    - '0.95'
    - -tp
    - '1'
    - --enforce-eager
    - --trust-remote-code


configParams:
api: completion
framework: vllm
max_model_len: 4096
max_tokens: 1024
ref_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '8000', --served-model-name, llm, --max-model-len,
      '4096', --trust-remote-code, --host, 0.0.0.0, --enforce-eager]
    env:
    - {name: VLLM_ALLOW_LONG_MAX_MODEL_LEN, value: 1}
sut_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '17097', --served-model-name, llm, --max-model-len,
      '4096', --trust-remote-code, --host, 0.0.0.0, --enforce-eager]
    env:
    - {name: VLLM_ALLOW_LONG_MAX_MODEL_LEN, value: 1}


configParams:
docker_image: git.modelhub.org.cn:9443/enginex/xc-llm-va16:26.03
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
api: completion
lang: zh
max_model_len: 2048
sut_config:
  values:
    gpu_num: 1
    env:
    - {name: MAX_MODEL_LEN, value: 2048}
    command: [/bin/bash, -ic, vllm serve /model --port 8000 --served-model-name llm
        --max-model-len 2048 --gpu-memory-utilization 0.9 --enforce-eager --trust-remote-code
        -tp 1 --host 0.0.0.0]
ref_config:
  values:
    cpu_num: 2
    gpu_num: 1
    env:
    - {name: MAX_MODEL_LEN, value: 2048}
    command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len,
      '2048', --enforce-eager, --trust-remote-code, -tp, '1']
model: llm


configParams:
docker_image: git.modelhub.org.cn:9443/enginex-hygon/vllm:0.9.2
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
max_model_len: 4096
api: completion
sut_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '20644', --served-model-name, llm, --max-model-len,
      '4096', --enforce-eager, --trust-remote-code, -tp, '1']
ref_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len,
      '4096', --enforce-eager, --trust-remote-code, -tp, '1']


configParams:
framework: vllm-016
lang: en
max_model_len: 4096
sut_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '80'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - --enforce-eager
    - --trust-remote-code
    - -tp
    - '1'
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '80'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - --enforce-eager
    - --trust-remote-code
    - -tp
    - '1'


framework: vllm-customized

configParams:
docker_image: harbor.4pd.io/hardcore-tech/cambricon-mlu370-pytorch:v25.01-20260204
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm-customized
api: completion
lang: en
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.1
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '8000', --served-model-name, llm, --max-model-len,
      '4096', --dtype, half, --gpu-memory-utilization, '0.95', -tp, '1', --enforce-eager,
      --trust-remote-code]
ref_config:
  gpu_num: 1
  values:
    command: [vllm, serve, /model, --port, '8000', --served-model-name, llm, --max-model-len,
      '4096', --dtype, auto, --gpu-memory-utilization, '0.95', -tp, '1', --enforce-eager,
      --trust-remote-code]


framework: vllm-mlu

configParams:
docker_image: harbor.4pd.io/hardcore-tech/cambricon-mlu370-pytorch:v25.01-torch2.5.0-torchmlu1.24.1-ubuntu22.04-py310
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm-mlu
sut_config:
  values:
    gpu_num: 1
    env:
    - {name: MAX_MODEL_LEN, value: 4096}
    command: [vllm, serve, /model, --port, '8000', --served-model-name, llm, --max-model-len,
      '4096', --trust-remote-code, --dtype, float16]
ref_config:
  values:
    cpu_num: 2
    gpu_num: 1
    env:
    - {name: MAX_MODEL_LEN, value: 4096}
    command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len,
      '4096', --trust-remote-code, --dtype, float16]


configParams:
framework: vllm_fix_tokenizer
docker_image: PLACEHOLDER_DOCKER_IMAGE
nv_docker_image: PLACEHOLDER_NV_DOCKER_IMAGE
api: completion
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.2
top_p: 0.9
lang: zh
max_model_len: 4096
sut_config:
  gpu_num: 1
  values:
    command: [/opt/entrypoint.sh, /model, --port, '8000', --served-model-name, llm,
      --max-model-len, '4096', --gpu-memory-utilization, '0.9', --enforce-eager, --trust-remote-code]
ref_config:
  gpu_num: 1
  values:
    command: [/opt/entrypoint.sh, /model, --port, '8000', --served-model-name, llm,
      --max-model-len, '4096', --dtype, float16, --gpu-memory-utilization, '0.95',
      -tp, '1', --enforce-eager, --trust-remote-code]


taskType: text-to-image-generation

framework: diffusers

configParams:
docker_image: harbor-contest.4pd.io/zhanghao/t2i:bi100-v0.3
nv_docker_image: harbor-contest.4pd.io/zhanghao/t2i:a100-v0.3
framework: diffusers
sut_config:
  gpu_num: 1
  values:
    env:
    - {name: DTYPE, value: fp16}
    command: [/workspace/entrypoint.sh, --port, '80']
ref_config:
  gpu_num: 1
  values:
    env:
    - {name: DTYPE, value: fp16}
    command: [/workspace/entrypoint.sh, --port, '80']


configParams:
framework: diffusers
sut_config:
  values:
    gpu_num: 1
    env:
    - {name: DTYPE, value: fp16}
    command: [/workspace/entrypoint.sh, --port, '80']
ref_config:
  values:
    gpu_num: 1
    env:
    - {name: DTYPE, value: fp16}
    command: [/workspace/entrypoint.sh, --port, '80']


configParams:
framework: diffusers
sut_config:
  values:
    gpu_num: 1
    env:
    - name: DTYPE
      value: fp16
    command:
    - /workspace/entrypoint.sh
    - --port
    - '80'
ref_config:
  values:
    gpu_num: 1
    env:
    - name: DTYPE
      value: fp16
    command:
    - /workspace/entrypoint.sh
    - --port
    - '80'


configParams:
classification_type: sentiment/emotion/toxicity/hate (根据文本分类的任务类型四选一)
framework: transformers
nv_framework: transformers
modelhub_options:
  srcRelativePath: leaderboard/modelHubXC/nlptown/bert-base-multilingual-uncased-sentiment
  mountPoint: /model
sut_config:
  gpu_num: 1
  values:
    env:
    - name: CONFIG_JSON
      value: |
        {
          "model_class": "AutoModelForSequenceClassification",
          "tokenizer_class": "AutoTokenizer",
          "torch_dtype": "auto"
        }
    command:
    - python3
    - transformers_server.py
ref_config:
  gpu_num: 1
  values:
    env:
    - name: CONFIG_JSON
      value: |
        {
          "model_class": "AutoModelForSequenceClassification",
          "tokenizer_class": "AutoTokenizer",
          "torch_dtype": "auto"
        }
    command:
    - python3
    - transformers_server.py


configParams:
framework: transformers
nv_framework: transformers
sut_config:
  gpu_num: 1
  values:
    env:
    - name: CONFIG_JSON
      value: |
        {
          "model_class": "AutoModelForImageClassification",
          "processer_class": "AutoImageProcessor",
          "torch_dtype": "auto"
        }
    command:
    - python3
    - transformers_server.py
ref_config:
  gpu_num: 1
  values:
    env:
    - name: CONFIG_JSON
      value: |
        {
          "model_class": "AutoModelForImageClassification",
          "processer_class": "AutoImageProcessor",
          "torch_dtype": "auto"
        }
    command:
    - python3
    - transformers_server.py


taskType: visual-multi-modal

configParams:
docker_image: PLACEHOLDER_DOCKER_IMAGE
nv_docker_image: PLACEHOLDER_DOCKER_IMAGE
framework: llamacpp
api: completion
max_tokens: 1024
max_model_len: 2048
sut_config:
  gpu_num: 1
  values:
    command:
    - /workspace/llama.cpp/build_ascend/bin/llama-server
    - --model
    - /model/Free_Sydney_V2_Mistral_7b.Q8_0.gguf
    - --alias
    - llm
    - --threads
    - '20'
    - --n-gpu-layers
    - '128'
    - --ctx-size
    - '2048'
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --jinja
    - --flash-attn
    - 'off'
ref_config:
  gpu_num: 1
  values:
    command:
    - /workspace/llama.cpp/build/bin/llama-server
    - --model
    - /model/Free_Sydney_V2_Mistral_7b.Q8_0.gguf
    - --alias
    - llm
    - --threads
    - '20'
    - --n-gpu-layers
    - '128'
    - --ctx-size
    - '2048'
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --jinja
    - --flash-attn
    - 'off'


configParams:
framework: llamacpp
api: chat
lang: zh
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.1
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command:
    - /workspace/llama.cpp/build_ascend/bin/llama-server
    - --model
    - /model/Gemma2-FT-ai-medical-chatbot.Q8_0.gguf
    - --alias
    - llm
    - --threads
    - '16'
    - --n-gpu-layers
    - '128'
    - --prio
    - '3'
    - --min_p
    - '0.01'
    - --ctx-size
    - '4096'
    - --host
    - 0.0.0.0
    - --port
    - '3316'
    - --jinja
    - --flash-attn
    - 'off'
ref_config:
  gpu_num: 1
  values:
    command:
    - /workspace/llama.cpp/build/bin/llama-server
    - --model
    - /model/Gemma2-FT-ai-medical-chatbot.Q8_0.gguf
    - --alias
    - llm
    - --threads
    - '16'
    - --n-gpu-layers
    - '128'
    - --prio
    - '3'
    - --min_p
    - '0.01'
    - --ctx-size
    - '4096'
    - --host
    - 0.0.0.0
    - --port
    - '80'
    - --jinja


configParams:
framework: llamacpp
max_model_len: 4096
sut_config:
  gpu_num: 1
  values:
    command:
    - /app/llama-server
    - --model
    - /model/glm-4-9b-chat-1m.Q8_0.gguf
    - --alias
    - llm
    - --threads
    - '20'
    - --n-gpu-layers
    - '128'
    - --ctx-size
    - '4096'
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --jinja
    - --flash-attn
    - 'off'
ref_config:
  gpu_num: 1
  values:
    command:
    - /workspace/llama.cpp/build/bin/llama-server
    - --model
    - /model/glm-4-9b-chat-1m.Q8_0.gguf
    - --alias
    - llm
    - --threads
    - '20'
    - --n-gpu-layers
    - '128'
    - --ctx-size
    - '4096'
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --jinja
    - --flash-attn
    - 'off'


configParams:
framework: llamacpp
max_model_len: 1024
sut_config:
  gpu_num: 1
  values:
    command:
    - /app/llama-server
    - --model
    - /model/Qwen3-14B-BF16.gguf
    - --alias
    - llm
    - --threads
    - '20'
    - --n-gpu-layers
    - '128'
    - --ctx-size
    - '4096'
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --jinja
    - --flash-attn
    - 'off'
ref_config:
  gpu_num: 1
  values:
    command:
    - /workspace/llama.cpp/build/bin/llama-server
    - --model
    - /model/Qwen3-14B-BF16.gguf
    - --alias
    - llm
    - --threads
    - '20'
    - --n-gpu-layers
    - '128'
    - --ctx-size
    - '4096'
    - --host
    - 0.0.0.0
    - --port
    - '8000'
    - --jinja
    - --flash-attn
    - 'off'


configParams:
docker_image: PLACEHOLDER_DOCKER_IMAGE
nv_docker_image: PLACEHOLDER_DOCKER_IMAGE
framework: vllm
api: completion
lang: en
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.1
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '20644'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - -tp
    - '1'
    - --enforce-eager
    - --trust-remote-code
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '80'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - -tp
    - '1'
    - --enforce-eager
    - --trust-remote-code


configParams:
framework: vllm
docker_image: PLACEHOLDER_DOCKER_IMAGE
nv_docker_image: PLACEHOLDER_NV_DOCKER_IMAGE
api: completion
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.2
top_p: 0.9
lang: zh
max_model_len: 2048
sut_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '8000'
    - --served-model-name
    - llm
    - --max-model-len
    - '2048'
    - --dtype
    - auto
    - --gpu-memory-utilization
    - '0.95'
    - -tp
    - '1'
    - --enforce-eager
    - --trust-remote-code
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '8000'
    - --served-model-name
    - llm
    - --max-model-len
    - '2048'
    - --dtype
    - auto
    - --gpu-memory-utilization
    - '0.95'
    - -tp
    - '1'
    - --enforce-eager
    - --trust-remote-code


configParams:
framework: vllm
max_model_len: 4096
sut_config:
  gpu_num: 1
  values:
    env:
    - name: SERVED_MODEL_NAME
      value: vlm
    command:
    - /workspace/launch_service
    - --port
    - '80'
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '8000'
    - --served-model-name
    - vlm
    - --max-model-len
    - '4096'
    - --enforce-eager
    - --trust-remote-code


configParams:
docker_image: git.modelhub.org.cn:9443/enginex-iluvatar-bi150/vllm:0.8.3
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
api: completion
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.2
top_p: 0.9
sut_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '80'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - --gpu-memory-utilization
    - '0.9'
    - --enforce-eager
    - --trust-remote-code
    - -tp
    - '1'
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '80'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - --enforce-eager
    - --trust-remote-code
    - -tp
    - '1'


configParams:
framework: vllm
nv_framework: vllm_v10
max_model_len: 4096
sut_config:
  gpu_num: 1
  values:
    command:
    - /opt/conda/bin/vllm
    - serve
    - /model
    - --port
    - '8898'
    - --served-model-name
    - vlm
    - --max-model-len
    - '4096'
    - --enforce-eager
    - --trust-remote-code
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '8000'
    - --served-model-name
    - vlm
    - --max-model-len
    - '4096'
    - --enforce-eager
    - --trust-remote-code


configParams:
framewok: vllm
max_model_len: 4096
sut_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --served-model-name
    - llm
    - --trust-remote-code
    - --max-model-len
    - '4096'
    - --enforce-eager
    - --gpu-memory-utilization
    - '0.5'
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --served-model-name
    - llm
    - --trust-remote-code
    - --max-model-len
    - '4096'
    - --enforce-eager


configParams:
framework: vllm
max_model_len: 4096
sut_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '20644'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - --enforce-eager
    - --trust-remote-code
    - -tp
    - '1'
ref_config:
  gpu_num: 1
  values:
    command:
    - vllm
    - serve
    - /model
    - --port
    - '80'
    - --served-model-name
    - llm
    - --max-model-len
    - '4096'
    - --enforce-eager
    - --trust-remote-code
    - -tp
    - '1'