From 965342bafeb0c99a3a607b3ae6826455e4f66d55 Mon Sep 17 00:00:00 2001
From: z3st <z3st@modelhub.org.cn>
Date: Fri, 24 Jul 2026 20:02:17 +0800
Subject: [PATCH] feat: focus on Llama models only, switch back to vllm
 framework

---
 main.py | 31 +++++++++++++++----------------
 1 file changed, 15 insertions(+), 16 deletions(-)

diff --git a/main.py b/main.py
index f180287..153bc43 100644
--- a/main.py
+++ b/main.py
@@ -47,7 +47,7 @@ SUPPORTED_SPECIAL_ARCHS = ['Eagle3Speculator', 'LlamaForCausalLMEagle3']
 MODELHUB_API = "https://modelhub.org.cn/api"
 
 # 搜索关键词
-SEARCH_KEYWORDS = ['qwen', 'Qwen2', 'Qwen3', 'Qwen3.5', 'Llama-3', 'Llama-3.1', 'Mistral', 'DeepSeek']
+SEARCH_KEYWORDS = ['Llama-3', 'Llama-3.1', 'Llama-3.2', 'Meta-Llama', 'Llama-4']
 
 # ============================================================
 # 全局状态
@@ -254,26 +254,25 @@ def check_queue_available() -> int:
 
 
 def build_config_params() -> str:
-    """构建 YAML 配置 - llama.cpp (支持 GGUF)"""
+    """构建 YAML 配置 - vllm"""
     params = {
-        'framework': 'llama.cpp',
-        'nv_framework': 'llama.cpp',
+        'framework': 'vllm',
+        'nv_framework': 'vllm',
         'api': 'completion',
         'max_tokens': 1024,
         'temperature': 0.7,
         'repetition_penalty': 1.2,
         'top_p': 0.9,
         'lang': 'zh',
-        'max_model_len': 4096,
+        'max_model_len': 2048,
         'sut_config': {
             'gpu_num': 1,
             'values': {
                 'command': [
-                    'llama-server', '--model', '/model', '--alias', 'llm',
-                    '--threads', '20', '--n-gpu-layers', '999', '--prio', '3',
-                    '--min_p', '0.01', '--ctx-size', '4096',
-                    '--host', '0.0.0.0', '--port', '8000',
-                    '--jinja', '--flash-attn', 'off',
+                    'vllm', 'serve', '/model', '--port', '8000',
+                    '--served-model-name', 'llm', '--max-model-len', '2048',
+                    '--dtype', 'auto', '--gpu-memory-utilization', '0.95',
+                    '-tp', '1', '--enforce-eager', '--trust-remote-code',
                 ]
             }
         },
@@ -281,9 +280,9 @@ def build_config_params() -> str:
             'gpu_num': 1,
             'values': {
                 'command': [
-                    'llama-server', '--model', '/model', '--alias', 'llm',
-                    '--threads', '20', '--n-gpu-layers', '999',
-                    '--ctx-size', '4096', '--host', '0.0.0.0', '--port', '8000',
+                    'vllm', 'serve', '/model', '--port', '80',
+                    '--served-model-name', 'llm', '--max-model-len', '4096',
+                    '--enforce-eager', '--trust-remote-code', '-tp', '1',
                 ]
             }
         },
@@ -303,7 +302,7 @@ def submit_model(model_url: str) -> tuple:
         'modelAddress': normalize_model_url(model_url),
         'taskType': 'text-generation',
         'targetGpu': TARGET_GPU,
-        'framework': 'llama.cpp',
+        'framework': 'vllm',
         'strategyId': STRATEGY_ID,
         'configParams': build_config_params(),
     }
@@ -567,7 +566,7 @@ class AgentHandler(BaseHTTPRequestHandler):
                     'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B',
                     'taskType': 'text-generation',
                     'targetGpu': TARGET_GPU,
-                    'framework': 'llama.cpp',
+                    'framework': 'vllm',
                     'strategyId': STRATEGY_ID,
                     'configParams': build_config_params(),
                 },
@@ -685,7 +684,7 @@ def main():
                 json={
                     'modelAddress': 'https://www.modelscope.cn/models/Qwen/Qwen3-8B',
                     'taskType': 'text-generation', 'targetGpu': TARGET_GPU,
-                    'framework': 'llama.cpp', 'strategyId': STRATEGY_ID,
+                    'framework': 'vllm', 'strategyId': STRATEGY_ID,
                     'configParams': build_config_params(),
                 }, timeout=10
             )