535 lines
16 KiB
Python
535 lines
16 KiB
Python
|
"""目标算力卡配置：卡名、平台内部 GPU 标识、默认框架与配置参数模板。
|
|||
|
|
|
|||
|
|
configParams 取自平台网页端选择「适配成功率最高」框架后自动生成的 YAML，
|
|||
|
|
保证与手工提交时下发到验证机的配置完全一致。
|
|||
|
|
"""
|
|||
|
|
|
|||
|
|
CARDS = [
|
|||
|
|
{
|
|||
|
|
"gpu": 'Biren_166m',
|
|||
|
|
"label": '壁砺166M',
|
|||
|
|
"framework": 'vllm_fix_tokenizer',
|
|||
|
|
"configParams": (
|
|||
|
|
"framework: vllm_fix_tokenizer\n"
|
|||
|
|
"nv_framework: vllm_fix_tokenizer\n"
|
|||
|
|
"lang: en\n"
|
|||
|
|
"max_model_len: 4096\n"
|
|||
|
|
"sut_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - /opt/entrypoint.sh - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '80'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
"ref_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - /opt/entrypoint.sh\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '8000'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --dtype\n"
|
|||
|
|
" - float16\n"
|
|||
|
|
" - --gpu-memory-utilization\n"
|
|||
|
|
" - '0.95'\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code"
|
|||
|
|
),
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"gpu": 'Iluvatar_mrv-100',
|
|||
|
|
"label": '智铠100',
|
|||
|
|
"framework": 'vllm',
|
|||
|
|
"configParams": (
|
|||
|
|
"framework: vllm\n"
|
|||
|
|
"nv_framework: vllm\n"
|
|||
|
|
"max_model_len: 4096\n"
|
|||
|
|
"lang: en\n"
|
|||
|
|
"temperature: 0.4\n"
|
|||
|
|
"repetition_penalty: 1.1\n"
|
|||
|
|
"top_p: 0.9\n"
|
|||
|
|
"sut_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - vllm\n"
|
|||
|
|
" - serve\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '20644'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --gpu-memory-utilization\n"
|
|||
|
|
" - '0.9'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
"ref_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - vllm\n"
|
|||
|
|
" - serve\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '80'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'"
|
|||
|
|
),
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"gpu": 'Iluvatar_bi-150',
|
|||
|
|
"label": '天垓150',
|
|||
|
|
"framework": 'vllm',
|
|||
|
|
"configParams": (
|
|||
|
|
"framework: vllm\n"
|
|||
|
|
"nv_framework: vllm\n"
|
|||
|
|
"api: completion\n"
|
|||
|
|
"max_model_len: 4096\n"
|
|||
|
|
"max_tokens: 1024\n"
|
|||
|
|
"temperature: 0.7\n"
|
|||
|
|
"repetition_penalty: 1.2\n"
|
|||
|
|
"top_p: 0.9\n"
|
|||
|
|
"sut_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - vllm\n"
|
|||
|
|
" - serve\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '80'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --gpu-memory-utilization\n"
|
|||
|
|
" - '0.9'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
"ref_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - vllm\n"
|
|||
|
|
" - serve\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '80'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'"
|
|||
|
|
),
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"gpu": 'Cambricon_mlu-370-x8',
|
|||
|
|
"label": 'MLU370-X8',
|
|||
|
|
"framework": 'vllm-mlu',
|
|||
|
|
"configParams": (
|
|||
|
|
"framework: vllm-mlu\n"
|
|||
|
|
"nv_framework: vllm_fix_tokenizer\n"
|
|||
|
|
"api: completion\n"
|
|||
|
|
"lang: en\n"
|
|||
|
|
"temperature: 0.4\n"
|
|||
|
|
"repetition_penalty: 1.1\n"
|
|||
|
|
"top_p: 0.9\n"
|
|||
|
|
"sut_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" env:\n"
|
|||
|
|
" - name: MAX_MODEL_LEN\n"
|
|||
|
|
" value: 4096\n"
|
|||
|
|
" - name: VLLM_ALLOW_LONG_MAX_MODEL_LEN\n"
|
|||
|
|
" value: '1'\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - vllm\n"
|
|||
|
|
" - serve\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '8000'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - --dtype\n"
|
|||
|
|
" - float16\n"
|
|||
|
|
"ref_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - /opt/entrypoint.sh\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '8000'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --dtype\n"
|
|||
|
|
" - float16\n"
|
|||
|
|
" - --gpu-memory-utilization\n"
|
|||
|
|
" - '0.95'\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code"
|
|||
|
|
),
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"gpu": 'Sunrise_pt-200-x1',
|
|||
|
|
"label": '曦望 S2',
|
|||
|
|
"framework": 'vllm_fix_tokenizer',
|
|||
|
|
"configParams": (
|
|||
|
|
"framework: vllm_fix_tokenizer\n"
|
|||
|
|
"nv_framework: vllm_fix_tokenizer\n"
|
|||
|
|
"api: completion\n"
|
|||
|
|
"max_tokens: 1024\n"
|
|||
|
|
"temperature: 0.7\n"
|
|||
|
|
"repetition_penalty: 1.2\n"
|
|||
|
|
"top_p: 0.9\n"
|
|||
|
|
"lang: zh\n"
|
|||
|
|
"max_model_len: 4096\n"
|
|||
|
|
"sut_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - /opt/entrypoint.sh\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '8000'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --gpu-memory-utilization\n"
|
|||
|
|
" - '0.9'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
"ref_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - /opt/entrypoint.sh\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '8000'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --dtype\n"
|
|||
|
|
" - float16\n"
|
|||
|
|
" - --gpu-memory-utilization\n"
|
|||
|
|
" - '0.95'\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code"
|
|||
|
|
),
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"gpu": 'Kunlunxin_p-800',
|
|||
|
|
"label": 'P800',
|
|||
|
|
"framework": 'vllm_fix_tokenizer',
|
|||
|
|
"configParams": (
|
|||
|
|
"framework: vllm_fix_tokenizer\n"
|
|||
|
|
"nv_framework: vllm_fix_tokenizer\n"
|
|||
|
|
"lang: en\n"
|
|||
|
|
"max_model_len: 4096\n"
|
|||
|
|
"sut_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - /opt/entrypoint.sh - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '80'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
"ref_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - /opt/entrypoint.sh\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '8000'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --dtype\n"
|
|||
|
|
" - float16\n"
|
|||
|
|
" - --gpu-memory-utilization\n"
|
|||
|
|
" - '0.95'\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code"
|
|||
|
|
),
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"gpu": 'MetaX_c-500',
|
|||
|
|
"label": '曦云C500',
|
|||
|
|
"framework": 'vllm_tokenizer_patch',
|
|||
|
|
"configParams": (
|
|||
|
|
"nv_framework: vllm_fix_tokenizer\n"
|
|||
|
|
"framework: vllm_tokenizer_patch\n"
|
|||
|
|
"storage: gpfs\n"
|
|||
|
|
"max_tokens: 1024\n"
|
|||
|
|
"max_model_len: 4096\n"
|
|||
|
|
"api: completion\n"
|
|||
|
|
"sut_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command: ['/opt/conda/bin/vllm', 'serve', '/model', '--port', '8000', '--served-model-name', 'llm', '--max-model-len', '4096', '--gpu-memory-utilization', '0.9', '--enforce-eager', '--trust-remote-code' ,'-tp', '1' ]\n"
|
|||
|
|
"ref_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - /opt/entrypoint.sh\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '8000'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --dtype\n"
|
|||
|
|
" - float16\n"
|
|||
|
|
" - --gpu-memory-utilization\n"
|
|||
|
|
" - '0.95'\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code"
|
|||
|
|
),
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"gpu": 'hygon_k100-ai',
|
|||
|
|
"label": 'K100AI',
|
|||
|
|
"framework": 'vllm',
|
|||
|
|
"configParams": (
|
|||
|
|
"framework: vllm\n"
|
|||
|
|
"nv_framework: vllm\n"
|
|||
|
|
"max_model_len: 4096\n"
|
|||
|
|
"api: completion\n"
|
|||
|
|
"sut_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - vllm\n"
|
|||
|
|
" - serve\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '20644'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
"ref_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - vllm\n"
|
|||
|
|
" - serve\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '80'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'"
|
|||
|
|
),
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"gpu": 'Vastai_va16',
|
|||
|
|
"label": 'VGX VA16',
|
|||
|
|
"framework": 'vllm_fix_tokenizer',
|
|||
|
|
"configParams": (
|
|||
|
|
"framework: vllm_fix_tokenizer\n"
|
|||
|
|
"nv_framework: vllm_fix_tokenizer\n"
|
|||
|
|
"lang: en\n"
|
|||
|
|
"max_model_len: 4096\n"
|
|||
|
|
"sut_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - /opt/entrypoint.sh\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '80'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
"ref_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - /opt/entrypoint.sh\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '8000'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --dtype\n"
|
|||
|
|
" - float16\n"
|
|||
|
|
" - --gpu-memory-utilization\n"
|
|||
|
|
" - '0.95'\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code"
|
|||
|
|
),
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"gpu": 'Iluvatar_bi-100',
|
|||
|
|
"label": '天垓100',
|
|||
|
|
"framework": 'vllm',
|
|||
|
|
"configParams": (
|
|||
|
|
"framework: vllm\n"
|
|||
|
|
"nv_framework: vllm\n"
|
|||
|
|
"lang: en\n"
|
|||
|
|
"max_model_len: 4096\n"
|
|||
|
|
"sut_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - vllm\n"
|
|||
|
|
" - serve\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '80'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --gpu-memory-utilization\n"
|
|||
|
|
" - '0.9'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
"ref_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - vllm\n"
|
|||
|
|
" - serve\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '80'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'"
|
|||
|
|
),
|
|||
|
|
},
|
|||
|
|
{
|
|||
|
|
"gpu": 'Cambricon_mlu-370-x4',
|
|||
|
|
"label": 'MLU370-X4',
|
|||
|
|
"framework": 'vllm-mlu',
|
|||
|
|
"configParams": (
|
|||
|
|
"framework: vllm-mlu\n"
|
|||
|
|
"nv_framework: vllm_fix_tokenizer\n"
|
|||
|
|
"sut_config:\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" env:\n"
|
|||
|
|
" - name: MAX_MODEL_LEN\n"
|
|||
|
|
" value: 4096\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - vllm\n"
|
|||
|
|
" - serve\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '8000'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --trust-remote-code\n"
|
|||
|
|
" - --dtype\n"
|
|||
|
|
" - float16\n"
|
|||
|
|
"ref_config:\n"
|
|||
|
|
" gpu_num: 1\n"
|
|||
|
|
" values:\n"
|
|||
|
|
" command:\n"
|
|||
|
|
" - /opt/entrypoint.sh\n"
|
|||
|
|
" - /model\n"
|
|||
|
|
" - --port\n"
|
|||
|
|
" - '8000'\n"
|
|||
|
|
" - --served-model-name\n"
|
|||
|
|
" - llm\n"
|
|||
|
|
" - --max-model-len\n"
|
|||
|
|
" - '4096'\n"
|
|||
|
|
" - --dtype\n"
|
|||
|
|
" - float16\n"
|
|||
|
|
" - --gpu-memory-utilization\n"
|
|||
|
|
" - '0.95'\n"
|
|||
|
|
" - -tp\n"
|
|||
|
|
" - '1'\n"
|
|||
|
|
" - --enforce-eager\n"
|
|||
|
|
" - --trust-remote-code"
|
|||
|
|
),
|
|||
|
|
},
|
|||
|
|
]
|
|||
|
|
|
|||
|
|
CARD_INDEX = {c["gpu"]: c for c in CARDS}
|