z3st/modelhub-submit-agent
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity

feat: switch to llama.cpp with GGUF support on Iluvatar, keep only GPTQ/AWQ filtered

Browse Source
This commit is contained in:
z3st
2026-07-24 19:29:48 +08:00
parent e0b6c3b5a8
commit e28f2e3eca
1 changed files with 18 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

35
main.py
View File

@@ -254,25 +254,26 @@ def check_queue_available() -> int:
def build_config_params() -> str:
"""构建 YAML 配置 - 完全匹配平台自动生成的格式（只支持vllm）"""
"""构建 YAML 配置 - llama.cpp (支持 GGUF)"""
params = {
'framework': 'vllm',
'nv_framework': 'vllm',
'framework': 'llama.cpp',
'nv_framework': 'llama.cpp',
'api': 'completion',
'max_tokens': 1024,
'temperature': 0.7,
'repetition_penalty': 1.2,
'top_p': 0.9,
'lang': 'zh',
'max_model_len': 2048,
'max_model_len': 4096,
'sut_config': {
'gpu_num': 1,
'values': {
'command': [
'vllm', 'serve', '/model', '--port', '8000',
'--served-model-name', 'llm', '--max-model-len', '2048',
'--dtype', 'auto', '--gpu-memory-utilization', '0.95',
'-tp', '1', '--enforce-eager', '--trust-remote-code',
'llama-server', '--model', '/model', '--alias', 'llm',
'--threads', '20', '--n-gpu-layers', '999', '--prio', '3',
'--min_p', '0.01', '--ctx-size', '4096',
'--host', '0.0.0.0', '--port', '8000',
'--jinja', '--flash-attn', 'off',
]
}
},
@@ -280,9 +281,9 @@ def build_config_params() -> str:
'gpu_num': 1,
'values': {
'command': [
'vllm', 'serve', '/model', '--port', '80',
'--served-model-name', 'llm', '--max-model-len', '4096',
'--enforce-eager', '--trust-remote-code', '-tp', '1',
'llama-server', '--model', '/model', '--alias', 'llm',
'--threads', '20', '--n-gpu-layers', '999',
'--ctx-size', '4096', '--host', '0.0.0.0', '--port', '8000',
]
}
},
@@ -302,7 +303,7 @@ def submit_model(model_url: str) -> tuple:
'modelAddress': normalize_model_url(model_url),
'taskType': 'text-generation',
'targetGpu': TARGET_GPU,
'framework': 'vllm',
'framework': 'llama.cpp',
'strategyId': STRATEGY_ID,
'configParams': build_config_params(),
}
@@ -347,11 +348,11 @@ def run_pipeline(submit_limit: int = 2):
time.sleep(0.3)
log(f"搜索完成: {len(seen)} 个唯一模型, {len(all_models)} 个下载量{DOWNLOAD_MIN}-{DOWNLOAD_MAX}")
# 2. 格式筛选（排除 GGUF/GPTQ/AWQ）
# 2. 格式筛选（排除 GPTQ/AWQ，保留 GGUF 和 HuggingFace）
log("\n--- 阶段2: 格式筛选 ---")
hf_models = []
format_skipped = 0
SKIP_FORMATS = ['GGUF', 'GPTQ', 'AWQ']
SKIP_FORMATS = ['GPTQ', 'AWQ']
for m in all_models:
mid_upper = m['model_id'].upper()
skip = False
@@ -363,7 +364,7 @@ def run_pipeline(submit_limit: int = 2):
break
if not skip:
hf_models.append(m)
log(f"格式筛选: {len(hf_models)} 通过, {format_skipped} 跳过 (GGUF/GPTQ/AWQ)")
log(f"格式筛选: {len(hf_models)} 通过, {format_skipped} 跳过 (GPTQ/AWQ)")
# 3. 架构筛选（排除 Qwen3.5 在 Iluvatar 上不支持）
log("\n--- 阶段3: 架构筛选 ---")
@@ -577,7 +578,7 @@ class AgentHandler(BaseHTTPRequestHandler):
'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B',
'taskType': 'text-generation',
'targetGpu': TARGET_GPU,
'framework': 'vllm',
'framework': 'llama.cpp',
'strategyId': STRATEGY_ID,
'configParams': build_config_params(),
},
@@ -695,7 +696,7 @@ def main():
json={
'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B',
'taskType': 'text-generation', 'targetGpu': TARGET_GPU,
'framework': 'vllm', 'strategyId': STRATEGY_ID,
'framework': 'llama.cpp', 'strategyId': STRATEGY_ID,
'configParams': build_config_params(),
}, timeout=10
)