""" xc_validation_strategy — 主入口 启动后针对 4 张 GPU 卡（Biren_166m / Cambricon_mlu-370-x8 / MetaX_c-500 / Kunlunxin_p-800）分别批量提交各自筛选出的模型验证任务（/adminApi/async/task/create-contest-task， Bearer Token 认证），之后保持 HTTP 服务存活。 同时暴露 /health（K8s 探活）和 /status（运行状态）。 """ import json import os import signal import threading from datetime import datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import List, Tuple import requests # ══════════════════════════════════════════════════════════ # 配置（全部从环境变量读取，不硬编码敏感信息） # ══════════════════════════════════════════════════════════ BASE_URL = os.environ.get("BASE_URL", "https://modelhub.org.cn") SUBMIT_ENDPOINT = "/adminApi/async/task/create-contest-task" # 通过 curl -X POST https://modelhub.org.cn/adminApi/user/login 获取后填入 AUTH_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyQWNjb3VudCI6Inpob3VzaGFzaGEiLCJpZCI6MTQsInVzZXJSb2xlIjoibGVhZGVyYm9hcmQiLCJleHAiOjE3ODU3NDY3NTMsImlhdCI6MTc4NTE0MTk1M30.KwUuefNAFSNwq3_Pnaw2nef8ZC6WgsECQ_LMeQnKk2c" CONTEST_API_TOKEN = "ef1ef82f3c9efee413d602345fbe224d" CONTRIBUTORS = "zhoushasha" TASK_TYPE = "text-generation" STRATEGY_ID = os.environ.get("STRATEGY_ID", "") # 平台自动注入，无需修改 HTTP_HOST = "0.0.0.0" HTTP_PORT = 8080 # ══════════════════════════════════════════════════════════ # 各 GPU 的模型列表（来自 filter_verified_models 脚本的筛选结果） # ══════════════════════════════════════════════════════════ BIREN_MODELS = [ "BigRatz/LOL-AI-2026", "aimeri/spoomplesmaxx-cardmaker-v1", "Alibaba-DT/Logics-STEM-8B-SFT", "Muneebmn123/insurance-voice-qwen25-1_5b", "AnkitBirGurung/NEMO-12B-SFT-Further", "danilarudenko/editorai-mini", "EphemeralYou/Prompt-Refine-MiniCPM5-1B", "mtepe01/mentorx-mistral-7b-automata-merged", "DarkArtsForge/Helix-SCE-12B-jh", "rpant/iolai26-solve", "NithinAI12/NithinX-Omni-LLM-v1", "ConvexAI/Luminex-34B-v0.2", "codellama/CodeLlama-34b-hf", "Lipas007/iol-ai-2026-qwen14b-awq", "D-Z-W/finetuned-teacher", "huan1999/ziya-llama-13b-medical-merged", "codellama/CodeLlama-34b-Python-hf", "ld4ad/gemma-2-9b-dunhuang", "harindhar10/Olmo-7b_1M_Smiles_lora", "allenai/Olmo-3-7B-Think-DPO", "allenai/Olmo-3-32B-Think-DPO", "RedHatAI/gemma-2-9b-it", "prashanthsura/gemma-2-2b-legal-financial-sft", "pfnet/plamo-2-8b", "sail/Sailor2-20B-128K-SFT", "facebook/layerskip-llama3.2-1B", "Qwen/Qwen2.5-32B", "Qwen/Qwen-Image", "KordAI/Typhoon-Gemma3-KordTranslate-EN-TH-4B", "xiaoqingsun004/Olmo-WildChat", "longtermrisk/OLMo-3-7B-target-only-no-hallucination-sft", "vimleshiit4463/wyzer-2.0-smollm2-135m", "trl-lib/pythia-1b-deduped-tldr-sft", ] CAMBRICON_MODELS = [ "clear-blue-sky/evolai-reborn-tfm-008", "clear-blue-sky/evolai-reborn-tfm-010", "BigRatz/LOL-AI-2026", "clear-blue-sky/evolai-reborn-tfm-001", "clear-blue-sky/evolai-reborn-tfm-019", "clear-blue-sky/evolai-reborn-tfm-007", "aimeri/spoomplesmaxx-cardmaker-v1", "aipatseer/inst_ft_qwen_0.6b_summ", "clear-blue-sky/evolai-reborn-tfm-003", "clear-blue-sky/evolai-reborn-tfm-004", "Alibaba-DT/Logics-STEM-8B-SFT", "clear-blue-sky/evolai-reborn-tfm-002", "prism-ml/Ternary-Bonsai-4B-unpacked", "Muneebmn123/insurance-voice-qwen25-1_5b", "AnkitBirGurung/NEMO-12B-SFT-Further", "danilarudenko/editorai-mini", "rpant/iolai26-solve", ] METAX_MODELS = [ "Phoenix9781/evolai-tf-model-105", "clear-blue-sky/evolai-reborn-tfm-008", "clear-blue-sky/evolai-reborn-tfm-010", "BigRatz/LOL-AI-2026", "clear-blue-sky/evolai-reborn-tfm-001", "clear-blue-sky/evolai-reborn-tfm-019", "clear-blue-sky/evolai-reborn-tfm-007", "clear-blue-sky/evolai-reborn-tfm-005", "Lin2es/evolai-tfm-03o", "clear-blue-sky/evolai-reborn-tfm-009", "aimeri/spoomplesmaxx-cardmaker-v1", "aipatseer/inst_ft_qwen_0.6b_summ", "reaperdoesntknow/DualMind-TKD-Agentic-1.7B", "clear-blue-sky/evolai-reborn-tfm-011", "clear-blue-sky/evolai-reborn-tfm-003", "clear-blue-sky/evolai-reborn-tfm-004", "clear-blue-sky/evolai-reborn-tfm-002", "amd/ReasonLite-0.6B", "prism-ml/Ternary-Bonsai-4B-unpacked", "Muneebmn123/insurance-voice-qwen25-1_5b", "danilarudenko/editorai-mini", "rpant/iolai26-solve", ] KUNLUNXIN_MODELS = [ "Patriae/patriae-cuban-dialect-model", "Phoenix9781/evolai-tf-model-105", "clear-blue-sky/evolai-reborn-tfm-008", "clear-blue-sky/evolai-reborn-tfm-010", "clear-blue-sky/evolai-reborn-tfm-001", "clear-blue-sky/evolai-reborn-tfm-019", "clear-blue-sky/evolai-reborn-tfm-007", "clear-blue-sky/evolai-reborn-tfm-005", "Lin2es/evolai-tfm-03o", "clear-blue-sky/evolai-reborn-tfm-009", "aimeri/spoomplesmaxx-cardmaker-v1", "aipatseer/inst_ft_qwen_0.6b_summ", "clear-blue-sky/evolai-reborn-tfm-011", "clear-blue-sky/evolai-reborn-tfm-003", "clear-blue-sky/evolai-reborn-tfm-004", "clear-blue-sky/evolai-reborn-tfm-002", "amd/ReasonLite-0.6B", "prism-ml/Ternary-Bonsai-4B-unpacked", "EthanGao123/CellHermes-v1.0", "RecursiveMAS/Mixture-Science-BioMistral-7B", "chenyitian-shanshu/SIRL-Gurobi", "RedHatAI/gemma-2-9b-it", "gaunernst/gemma-3-27b-it-qat-autoawq", "promotion/qwen3-8b-simpo-avg-b2p5-g1p0-s42", "llamaindex/vdr-2b-multi-v1", "KordAI/Typhoon-Gemma3-KordTranslate-EN-TH-4B", "TheDrummer/UnslopNemo-12B-v3", "gradients-io-tournaments/tournament-tourn_c5d86c82ce819a79_20260706-b78a01d4-0a6a-49e1-9190-5e88ae329937-5DS6XMVr", "ArliAI/Mistral-Nemo-12B-ArliAI-RPMax-v1.1", ] # 按顺序处理：Biren → Cambricon → MetaX → Kunlunxin GPU_JOBS: List[Tuple[str, List[str]]] = [ ("Biren_166m", BIREN_MODELS), ("Cambricon_mlu-370-x8", CAMBRICON_MODELS), ("MetaX_c-500", METAX_MODELS), ("Kunlunxin_p-800", KUNLUNXIN_MODELS), ] TOTAL_MODELS = sum(len(models) for _, models in GPU_JOBS) # ══════════════════════════════════════════════════════════ # 全局状态（供 /status 展示） # ══════════════════════════════════════════════════════════ _state = { "strategy_id": STRATEGY_ID, "phase": "starting", # starting | submitting | done | error "total": TOTAL_MODELS, "submitted": 0, "failed": 0, "per_gpu": {gpu: 0 for gpu, _ in GPU_JOBS}, "started_at": None, "finished_at": None, } _shutdown = threading.Event() # ══════════════════════════════════════════════════════════ # HTTP 服务 # ══════════════════════════════════════════════════════════ class Handler(BaseHTTPRequestHandler): def do_GET(self): if self.path == "/health": self._json({"status": "ok"}) elif self.path == "/status": self._json(_state) else: self._json({"error": "not found"}, 404) def _json(self, body: dict, code: int = 200): payload = json.dumps(body, default=str).encode() self.send_response(code) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(payload))) self.end_headers() self.wfile.write(payload) def log_message(self, fmt, *args): print(f"[http] {self.address_string()} {fmt % args}", flush=True) def _run_http(): server = ThreadingHTTPServer((HTTP_HOST, HTTP_PORT), Handler) server.timeout = 1 print(f"[http] 监听 {HTTP_HOST}:{HTTP_PORT}", flush=True) while not _shutdown.is_set(): server.handle_request() server.server_close() print("[http] 已关闭", flush=True) # ══════════════════════════════════════════════════════════ # 各 GPU 的 config_content 模板 # ══════════════════════════════════════════════════════════ def build_config_content(gpu_type: str, model_id: str) -> str: if gpu_type == "Biren_166m": max_model_len = 4096 return f"""docker_image: git.modelhub.org.cn:9443/enginex/xc-llm-biren166m:26.01 nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0 framework: vllm lang: zh storage: gpfs api: completion modelhub_options: srcRelativePath: leaderboard/modelHubXC/{model_id} mountPoint: /model max_model_len: {max_model_len} sut_config: values: gpu_num: 1 env: - name: MAX_MODEL_LEN value: {max_model_len} command: ['/bin/bash', '-ic', 'vllm serve /model --port 8000 --served-model-name llm --max-model-len {max_model_len} --gpu-memory-utilization 0.9 --enforce-eager --trust-remote-code -tp 1 --host 0.0.0.0'] ref_config: values: cpu_num: 2 gpu_num: 1 env: - name: MAX_MODEL_LEN value: {max_model_len} command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '{max_model_len}', '--enforce-eager', '--trust-remote-code', '-tp', '1'] model: llm """ elif gpu_type == "Cambricon_mlu-370-x8": return f"""docker_image: harbor.4pd.io/hardcore-tech/cambricon-mlu370-pytorch:v25.01-torch2.5.0-torchmlu1.24.1-ubuntu22.04-py310 nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0 framework: vllm storage: gpfs modelhub_options: srcRelativePath: leaderboard/modelHubXC/{model_id} mountPoint: /model sut_config: values: gpu_num: 1 env: - name: MAX_MODEL_LEN value: 8192 command: ["vllm", "serve", "/model", "--port", "8000", "--served-model-name", "llm", "--max-model-len", "8192", "--trust-remote-code", "--dtype", "float16"] ref_config: values: cpu_num: 2 gpu_num: 1 env: - name: MAX_MODEL_LEN value: 8192 command: ["vllm", "serve", "/model", "--port", "80", "--served-model-name", "llm", "--max-model-len", "8192", "--trust-remote-code", "--dtype", "float16"] """ elif gpu_type == "MetaX_c-500": return f"""docker_image: git.modelhub.org.cn:9443/enginex-metax/vllm:0.9.1 nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0 framework: vllm lang: en storage: gpfs api: chat modelhub_options: srcRelativePath: leaderboard/modelHubXC/{model_id} mountPoint: /model max_model_len: 2048 sut_config: gpu_num: 1 values: command: ['/opt/conda/bin/vllm', 'serve', '/model', '--port', '20644', '--served-model-name', 'llm', '--max-model-len', '2048', '--gpu-memory-utilization', '0.9', '--enforce-eager', '--trust-remote-code' ,'-tp', '1'] ref_config: gpu_num: 1 values: command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '2048', '--enforce-eager', '--trust-remote-code', '-tp', '1'] """ elif gpu_type == "Kunlunxin_p-800": return f"""docker_image: git.modelhub.org.cn:9443/enginex/xc-llm-kunlun nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0 framework: vllm lang: en storage: gpfs api: chat temperature: 0.4 repetition_penalty: 1.1 top_p: 0.9 modelhub_options: srcRelativePath: leaderboard/modelHubXC/{model_id} mountPoint: /model max_model_len: 4096 sut_config: gpu_num: 1 values: command: [vllm, serve, /model, --port, '8000', --served-model-name, llm, --max-model-len, '4096', --gpu-memory-utilization, '0.9', --enforce-eager, --trust-remote-code, -tp, '1'] ref_config: gpu_num: 1 values: command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len, '4096', --enforce-eager, --trust-remote-code, -tp, '1'] """ else: raise ValueError(f"未知的 GPU_TYPE: {gpu_type}") # ══════════════════════════════════════════════════════════ # 业务逻辑 # ══════════════════════════════════════════════════════════ def _submit_task(token: str, gpu_type: str, model_id: str) -> Tuple[bool, str]: headers = { "Content-Type": "application/json", "Authorization": f"Bearer {token}", } config_content = build_config_content(gpu_type, model_id) payload = { "contestApiToken": CONTEST_API_TOKEN, "contributors": CONTRIBUTORS, "gpuTypes": [gpu_type], "taskType": TASK_TYPE, "modelId": model_id, "framework": "vllm", "strategyId": STRATEGY_ID, # 平台要求 "submissionConfig": [{ "config": config_content, "gpuType": gpu_type, "taskType": TASK_TYPE, }], } print(f"[payload] gpu={gpu_type} model={model_id}", flush=True) try: resp = requests.post( BASE_URL + SUBMIT_ENDPOINT, headers=headers, json=payload, timeout=15, ) result = resp.json() if result.get("code") == 0: task_id = result.get("data", {}).get("id", "") print(f"[worker] OK {model_id} (GPU={gpu_type}) task_id={task_id}", flush=True) return True, task_id else: print(f"[worker] FAIL {model_id} (GPU={gpu_type}): {result.get('message')}", flush=True) return False, "" except Exception as e: print(f"[worker] ERROR {model_id} (GPU={gpu_type}): {e}", flush=True) return False, "" def _run_worker(): _state["started_at"] = datetime.utcnow().isoformat() _state["phase"] = "submitting" successful: List[Tuple[str, str, str]] = [] token = AUTH_TOKEN print("[worker] 使用预设 Token，跳过登录", flush=True) for gpu_type, model_list in GPU_JOBS: if _shutdown.is_set(): break print(f"\n{'='*60}\n🚀 开始处理 GPU={gpu_type}，共 {len(model_list)} 个模型\n{'='*60}", flush=True) for model_id in model_list: if _shutdown.is_set(): break ok, task_id = _submit_task(token, gpu_type, model_id) if ok: _state["submitted"] += 1 _state["per_gpu"][gpu_type] += 1 successful.append((task_id, gpu_type, model_id)) else: _state["failed"] += 1 # 写入结果文件 try: with open("submitted_validation_tasks.txt", "w", encoding="utf-8") as f: for tid, gpu, mid in successful: f.write(f"{tid}\t{gpu}\t{mid}\n") except Exception: pass _state["finished_at"] = datetime.utcnow().isoformat() _state["phase"] = "done" print( f"[worker] 完成 submitted={_state['submitted']} failed={_state['failed']} " f"total={_state['total']} per_gpu={_state['per_gpu']}", flush=True, ) # 提交完成后继续保持进程存活，等待平台停止 # ══════════════════════════════════════════════════════════ # 入口 # ══════════════════════════════════════════════════════════ def _handle_signal(signum, _frame): print(f"[main] 收到信号 {signum}，正在关闭...", flush=True) _shutdown.set() def main(): signal.signal(signal.SIGTERM, _handle_signal) signal.signal(signal.SIGINT, _handle_signal) # HTTP 服务线程 http_thread = threading.Thread(target=_run_http, daemon=False) http_thread.start() # 提交任务线程 worker_thread = threading.Thread(target=_run_worker, daemon=True) worker_thread.start() # 主线程等待 shutdown _shutdown.wait() print("[main] 等待 HTTP 服务关闭...", flush=True) http_thread.join(timeout=5) print("[main] 退出", flush=True) if __name__ == "__main__": main()