z3st/modelhub-submit-agent
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: switch to llama.cpp with GGUF support on Iluvatar, keep only GPTQ/AWQ filtered

Browse Source
This commit is contained in:
z3st
2026-07-24 19:29:48 +08:00
parent e0b6c3b5a8
commit e28f2e3eca
1 changed files with 18 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
main.py
View File

@@ -254,25 +254,26 @@ def check_queue_available() -> int:
def build_config_params() -> str: def build_config_params() -> str:
"""构建 YAML 配置 - 完全匹配平台自动生成的格式（只支持vllm）""" """构建 YAML 配置 - llama.cpp (支持 GGUF)"""
params = { params = {
'framework': 'vllm', 'framework': 'llama.cpp',
'nv_framework': 'vllm', 'nv_framework': 'llama.cpp',
'api': 'completion', 'api': 'completion',
'max_tokens': 1024, 'max_tokens': 1024,
'temperature': 0.7, 'temperature': 0.7,
'repetition_penalty': 1.2, 'repetition_penalty': 1.2,
'top_p': 0.9, 'top_p': 0.9,
'lang': 'zh', 'lang': 'zh',
'max_model_len': 2048, 'max_model_len': 4096,
'sut_config': { 'sut_config': {
'gpu_num': 1, 'gpu_num': 1,
'values': { 'values': {
'command': [ 'command': [
'vllm', 'serve', '/model', '--port', '8000', 'llama-server', '--model', '/model', '--alias', 'llm',
'--served-model-name', 'llm', '--max-model-len', '2048', '--threads', '20', '--n-gpu-layers', '999', '--prio', '3',
'--dtype', 'auto', '--gpu-memory-utilization', '0.95', '--min_p', '0.01', '--ctx-size', '4096',
'-tp', '1', '--enforce-eager', '--trust-remote-code', '--host', '0.0.0.0', '--port', '8000',
'--jinja', '--flash-attn', 'off',
] ]
} }
}, },
@@ -280,9 +281,9 @@ def build_config_params() -> str:
'gpu_num': 1, 'gpu_num': 1,
'values': { 'values': {
'command': [ 'command': [
'vllm', 'serve', '/model', '--port', '80', 'llama-server', '--model', '/model', '--alias', 'llm',
'--served-model-name', 'llm', '--max-model-len', '4096', '--threads', '20', '--n-gpu-layers', '999',
'--enforce-eager', '--trust-remote-code', '-tp', '1', '--ctx-size', '4096', '--host', '0.0.0.0', '--port', '8000',
] ]
} }
}, },
@@ -302,7 +303,7 @@ def submit_model(model_url: str) -> tuple:
'modelAddress': normalize_model_url(model_url), 'modelAddress': normalize_model_url(model_url),
'taskType': 'text-generation', 'taskType': 'text-generation',
'targetGpu': TARGET_GPU, 'targetGpu': TARGET_GPU,
'framework': 'vllm', 'framework': 'llama.cpp',
'strategyId': STRATEGY_ID, 'strategyId': STRATEGY_ID,
'configParams': build_config_params(), 'configParams': build_config_params(),
} }
@@ -347,11 +348,11 @@ def run_pipeline(submit_limit: int = 2):
time.sleep(0.3) time.sleep(0.3)
log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量{DOWNLOAD_MIN}-{DOWNLOAD_MAX}") log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量{DOWNLOAD_MIN}-{DOWNLOAD_MAX}")
# 2. 格式筛选（排除 GGUF/GPTQ/AWQ） # 2. 格式筛选（排除 GPTQ/AWQ，保留 GGUF 和 HuggingFace）
log("\n--- 阶段2: 格式筛选 ---") log("\n--- 阶段2: 格式筛选 ---")
hf_models = [] hf_models = []
format_skipped = 0 format_skipped = 0
SKIP_FORMATS = ['GGUF', 'GPTQ', 'AWQ'] SKIP_FORMATS = ['GPTQ', 'AWQ']
for m in all_models: for m in all_models:
mid_upper = m['model_id'].upper() mid_upper = m['model_id'].upper()
skip = False skip = False
@@ -363,7 +364,7 @@ def run_pipeline(submit_limit: int = 2):
break break
if not skip: if not skip:
hf_models.append(m) hf_models.append(m)
log(f"格式筛选: {len(hf_models)} 通过, {format_skipped} 跳过 (GGUF/GPTQ/AWQ)") log(f"格式筛选: {len(hf_models)} 通过, {format_skipped} 跳过 (GPTQ/AWQ)")
# 3. 架构筛选（排除 Qwen3.5 在 Iluvatar 上不支持） # 3. 架构筛选（排除 Qwen3.5 在 Iluvatar 上不支持）
log("\n--- 阶段3: 架构筛选 ---") log("\n--- 阶段3: 架构筛选 ---")
@@ -577,7 +578,7 @@ class AgentHandler(BaseHTTPRequestHandler):
'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B', 'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B',
'taskType': 'text-generation', 'taskType': 'text-generation',
'targetGpu': TARGET_GPU, 'targetGpu': TARGET_GPU,
'framework': 'vllm', 'framework': 'llama.cpp',
'strategyId': STRATEGY_ID, 'strategyId': STRATEGY_ID,
'configParams': build_config_params(), 'configParams': build_config_params(),
}, },
@@ -695,7 +696,7 @@ def main():
json={ json={
'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B', 'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B',
'taskType': 'text-generation', 'targetGpu': TARGET_GPU, 'taskType': 'text-generation', 'targetGpu': TARGET_GPU,
'framework': 'vllm', 'strategyId': STRATEGY_ID, 'framework': 'llama.cpp', 'strategyId': STRATEGY_ID,
'configParams': build_config_params(), 'configParams': build_config_params(),
}, timeout=10 }, timeout=10
) )