z3st/modelhub-submit-agent
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: focus on Llama models only, switch back to vllm framework

Browse Source
This commit is contained in:
z3st
2026-07-24 20:02:17 +08:00
parent 5bb32e6bdb
commit 965342bafe
1 changed files with 15 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
main.py
View File

@@ -47,7 +47,7 @@ SUPPORTED_SPECIAL_ARCHS = ['Eagle3Speculator', 'LlamaForCausalLMEagle3']
MODELHUB_API = "https://modelhub.org.cn/api" MODELHUB_API = "https://modelhub.org.cn/api"
# 搜索关键词 # 搜索关键词
SEARCH_KEYWORDS = ['qwen', 'Qwen2', 'Qwen3', 'Qwen3.5', 'Llama-3', 'Llama-3.1', 'Mistral', 'DeepSeek'] SEARCH_KEYWORDS = ['Llama-3', 'Llama-3.1', 'Llama-3.2', 'Meta-Llama', 'Llama-4']
# ============================================================ # ============================================================
# 全局状态 # 全局状态
@@ -254,26 +254,25 @@ def check_queue_available() -> int:
def build_config_params() -> str: def build_config_params() -> str:
"""构建 YAML 配置 - llama.cpp (支持 GGUF)""" """构建 YAML 配置 - vllm"""
params = { params = {
'framework': 'llama.cpp', 'framework': 'vllm',
'nv_framework': 'llama.cpp', 'nv_framework': 'vllm',
'api': 'completion', 'api': 'completion',
'max_tokens': 1024, 'max_tokens': 1024,
'temperature': 0.7, 'temperature': 0.7,
'repetition_penalty': 1.2, 'repetition_penalty': 1.2,
'top_p': 0.9, 'top_p': 0.9,
'lang': 'zh', 'lang': 'zh',
'max_model_len': 4096, 'max_model_len': 2048,
'sut_config': { 'sut_config': {
'gpu_num': 1, 'gpu_num': 1,
'values': { 'values': {
'command': [ 'command': [
'llama-server', '--model', '/model', '--alias', 'llm', 'vllm', 'serve', '/model', '--port', '8000',
'--threads', '20', '--n-gpu-layers', '999', '--prio', '3', '--served-model-name', 'llm', '--max-model-len', '2048',
'--min_p', '0.01', '--ctx-size', '4096', '--dtype', 'auto', '--gpu-memory-utilization', '0.95',
'--host', '0.0.0.0', '--port', '8000', '-tp', '1', '--enforce-eager', '--trust-remote-code',
'--jinja', '--flash-attn', 'off',
] ]
} }
}, },
@@ -281,9 +280,9 @@ def build_config_params() -> str:
'gpu_num': 1, 'gpu_num': 1,
'values': { 'values': {
'command': [ 'command': [
'llama-server', '--model', '/model', '--alias', 'llm', 'vllm', 'serve', '/model', '--port', '80',
'--threads', '20', '--n-gpu-layers', '999', '--served-model-name', 'llm', '--max-model-len', '4096',
'--ctx-size', '4096', '--host', '0.0.0.0', '--port', '8000', '--enforce-eager', '--trust-remote-code', '-tp', '1',
] ]
} }
}, },
@@ -303,7 +302,7 @@ def submit_model(model_url: str) -> tuple:
'modelAddress': normalize_model_url(model_url), 'modelAddress': normalize_model_url(model_url),
'taskType': 'text-generation', 'taskType': 'text-generation',
'targetGpu': TARGET_GPU, 'targetGpu': TARGET_GPU,
'framework': 'llama.cpp', 'framework': 'vllm',
'strategyId': STRATEGY_ID, 'strategyId': STRATEGY_ID,
'configParams': build_config_params(), 'configParams': build_config_params(),
} }
@@ -567,7 +566,7 @@ class AgentHandler(BaseHTTPRequestHandler):
'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B', 'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B',
'taskType': 'text-generation', 'taskType': 'text-generation',
'targetGpu': TARGET_GPU, 'targetGpu': TARGET_GPU,
'framework': 'llama.cpp', 'framework': 'vllm',
'strategyId': STRATEGY_ID, 'strategyId': STRATEGY_ID,
'configParams': build_config_params(), 'configParams': build_config_params(),
}, },
@@ -685,7 +684,7 @@ def main():
json={ json={
'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B', 'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B',
'taskType': 'text-generation', 'targetGpu': TARGET_GPU, 'taskType': 'text-generation', 'targetGpu': TARGET_GPU,
'framework': 'llama.cpp', 'strategyId': STRATEGY_ID, 'framework': 'vllm', 'strategyId': STRATEGY_ID,
'configParams': build_config_params(), 'configParams': build_config_params(),
}, timeout=10 }, timeout=10
) )