zhoushasha/xc_validation_strategy
1
0
Fork 0
Code Issues Pull Requests Actions Projects Releases Wiki Activity
Files
793c49aea336fd6c244a6278e58b11eea2a042a2
xc_validation_strategy/main.py
zhousha 793c49aea3 submit round-24 filter results on zhoushasha: MetaX_c-500(63)/Kunlunxin_p-800(1)/Cambricon_mlu-370-x8(1)/Biren_166m(95) + ppu_zw_810e(57), 217 models total 
Also add a hygon_k100-ai config branch and HYGON_MODELS list (kept available but
excluded from GPU_JOBS per request). Refresh the expired AUTH_TOKEN.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-15 17:25:35 +08:00

686 lines
27 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""
xc_validation_strategy — 主入口
启动后针对 GPU_JOBS 中配置的 GPU 卡分别批量提交各自筛选出的模型验证任务
（当前仅提交 ppu_zw_810e，其余 4 张卡 Biren_166m/Cambricon_mlu-370-x8/MetaX_c-500/
Kunlunxin_p-800 的 config_content 模板和模型列表仍保留在代码中，未列入本次 GPU_JOBS）
（/adminApi/async/task/create-contest-task，
Bearer Token 认证），之后保持 HTTP 服务存活。
账号额度自动重试：如果某个模型提交时命中"当前等待中或运行中的异步模型验证
任务数量已达上限"（账号额度已满），不算永久失败，会被留到下一轮；额度耗尽后
本进程会原地等待 30 分钟，再自动重试所有因额度问题未提交成功的模型，如此循环，
直至全部提交成功或进程被平台关闭——不需要重新部署新策略，循环逻辑在本进程内完成。
非额度原因的失败（如模型已在验证中等）不会重试。
同时暴露 /health（K8s 探活）和 /status（运行状态，含当前轮次/待重试数/下次重试时间）。
"""
import json
import os
import signal
import threading
from datetime import datetime
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import List, Tuple
import requests
# ══════════════════════════════════════════════════════════
# 配置（全部从环境变量读取，不硬编码敏感信息）
# ══════════════════════════════════════════════════════════
BASE_URL = os.environ.get("BASE_URL", "https://modelhub.org.cn")
SUBMIT_ENDPOINT = "/adminApi/async/task/create-contest-task"
# 通过 curl -X POST https://modelhub.org.cn/adminApi/user/login 获取后填入
AUTH_TOKEN = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyQWNjb3VudCI6Inpob3VzaGFzaGEiLCJpZCI6MTQsInVzZXJSb2xlIjoibGVhZGVyYm9hcmQiLCJleHAiOjE3OTAwNjkxMjAsImlhdCI6MTc4OTQ2NDMyMH0.KzJac6ddaZdtLvjD6ZnoK1PNEKFoXdyDn9Hh4FxU9ic"
CONTEST_API_TOKEN = "ef1ef82f3c9efee413d602345fbe224d"
CONTRIBUTORS = "zhoushasha"
TASK_TYPE = "text-generation"
STRATEGY_ID = os.environ.get("STRATEGY_ID", "") # 平台自动注入，无需修改
HTTP_HOST = "0.0.0.0"
HTTP_PORT = 8080
# ══════════════════════════════════════════════════════════
# 各 GPU 的模型列表（来自 filter_verified_models 脚本的筛选结果）
# ══════════════════════════════════════════════════════════
BIREN_MODELS = [
"ApolloRaines/Phi-4-mini-Instruct-Desyced",
"allenai/OLMo-2-0425-1B",
"Free2035/4QDR_4B_AD_Thinker_V1",
"aifoundry-org/OLMo-7B-0424-hf-Quantized",
"lugman-madhiai/Qwen3-4B-MHS-1.1",
"zypchn/BehChat-SFT-v4",
"NoesisLab/Kai-30B-Instruct",
"barandinho/Qwen3-30B-A3B-FIRST-STAGE-SFT",
"m-a-p/OpenLLaMA-Reproduce-218.1B",
"xiaolesu/Qwen3-8B-Herald-SFT",
"WhiteRabbitNeo/WhiteRabbitNeo-33B-v1.5",
"saleh1312/orph_3.07225",
"vanta-research/atom-olmo3-7b",
"ZhipuAI/LongCite-glm4-9b",
"Xlnk/LFM2-2.6B-Exp-GGuf",
"Tesslate/UIGEN-T1.1-Qwen-14B",
"jondurbin/bagel-dpo-34b-v0.2",
"zhengr/MixTAO-7Bx2-MoE-Instruct-v5.0",
"allenai/Olmo-3.1-32B-Instruct",
"bjaidi/Phi-3-medium-128k-instruct-awq",
"xing720310/qwen3-14b",
"IntelLabs/sqft-mistral-7b-v0.3-50-base-gptq",
"hariharanv04/qwen2.5-coder-14b-metadata-merged",
"junfengzhou/qwen3-14b-rl",
"ronnywebdevs1/Affine-P011-5CkU7wLMWXPs6TdSsMf8eEYCVAbPLyNmYg9PPx1Uds8toKra",
"kennedyantonio0301/Affine-Tensor-h3-5EkdoaCmEpFffUjDpLhDMzEDR4kptaEzpTPYCP1uL2sbct8C",
"julep-ai/dolphin-2.9-llama3-70b-awq",
"cortexso/gemma3",
"jacob-ml/jacob-24b",
"lyraaaa/neuralese-sft-pretrain-v2",
"ai-sage/GigaChat-20B-A3B-instruct-bf16",
"PJMixers-Dev/gemma-3-1b-it-fixed",
"commotion/svara_finetune_v1",
"geoffmunn/Qwen3-14B-f16",
"geoffmunn/Qwen3-32B-f16",
"TeichAI/Nemotron-Cascade-14B-Thinking-Claude-4.5-Opus-High-Reasoning-Distill",
"prithivMLmods-llamafile/SmolLM2-1.7B-Instruct-llamafile",
"prithivMLmods-llamafile/Llama-3.2-8B-llamafile-200K",
"llamafile-club/SmolLM-135M-Instruct-Llamafile",
"prithivMLmods/Sombrero-QwQ-32B-Elite9",
"prithivMLmods-llamafile/Aya-Expanse-8B-llamafile",
"llamafile-club/SmolLM-135M-Llamafile",
"prithivMLmods/Sombrero-QwQ-32B-Elite10-Fixed",
"prithivMLmods-llamafile/Qwen2.5-Coder-1.5B-llamafile",
"TeichAI/Qwen3-14B-Polaris-Alpha-Distill",
"okwinds/MiroThinker-14B-DPO-v0.1",
"sanbuphy/tianji-wish2-14b",
"codefuse-ai/CodeFuse-StarCoder2-15B",
"AI-ModelScope/txgemma-27b-chat",
"Shanghai_AI_Laboratory/internlm3-8b-instruct-awq",
"XGenerationLab/XiYanSQL-QwenCoder-32B-2412",
"vllm-ascend/gemma-1.1-2b-it",
"OpenBuddy/openbuddy-qwen1.5-32b-v21.2-32k",
"OpenBuddy/openbuddy-thinker-32b-v26-preview",
"OpenBuddy/openbuddy-qwen1.5-32b-v21.1-32k",
"TechxGenus-MS/CodeGemma-7b",
"OpenBuddy/openbuddy-qwq-32b-v25.2q-200k",
"unsloth/Qwen3-30B-A3B",
"OpenBuddy/openbuddy-qwq-32b-v25.1-200k",
"OpenBuddy/openbuddy-r1-32b-v24.1-200k",
"iic/ERank-14B",
"OpenBuddy/openbuddy-yi1.5-34b-v21.2-32k",
"LGAI-EXAONE/EXAONE-Deep-32B",
"OpenBuddy/openbuddy-qwq-32b-v24.2-200k",
"unsloth/Phi-3-mini-4k-instruct-v0",
"argilla/notux-8x7b-v1",
"voidful/qd-phi-1_5",
"Shanghai_AI_Laboratory/internlm2-math-base-20b",
"Shanghai_AI_Laboratory/internlm2-math-plus-20b",
"TechxGenus-MS/starcoder2-15b-instruct",
"Shanghai_AI_Laboratory/internlm2-base-20b",
"m-a-p/OpenLLaMA-Reproduce-872.42B",
"m-a-p/OpenLLaMA-Reproduce-973.08B",
"Shanghai_AI_Laboratory/OREAL-32B",
"YOYO-AI/Qwen3-30B-A3B-CoderThinking-YOYO-linear",
"ticoAg/Qwen-1_8B-Chat-Int4-awq",
"smirki/UIGEN-T1.1-Qwen-14B",
"prithivMLmods/Qwen2.5-32B-DeepSeek-R1-Instruct",
"sail/Sailor2-20B-128K",
"xverse/XVERSE-65B",
"Shanghai_AI_Laboratory/internlm2_5-20b-chat",
"TeleAI/TeleChat-52B",
"modelscope/Llama-2-70b-ms",
"Shanghai_AI_Laboratory/internlm2-20b",
"ai-modelscope/Llama-3_1-Nemotron-51B-Instruct",
"zhuangxialie/Phi-3-Chinese-ORPO",
"openai-mirror/gpt-oss-safeguard-20b",
"ByteDance-Seed/Seed-OSS-36B-Instruct",
"Shanghai_AI_Laboratory/internlm-chat-20b",
"TurkuNLP/bloom-finnish-176b",
"openai-mirror/gpt-oss-120b",
"vllm-ascend/QwQ-32B-W8A8",
"mistralai/Mistral-Small-24B-Instruct-2501",
"Shanghai_AI_Laboratory/internlm-20b",
"ZhipuAI/GLM-4-32B-0414",
]
CAMBRICON_MODELS = [
"Xlnk/LFM2-2.6B-Exp-GGuf",
]
METAX_MODELS = [
"ApolloRaines/Phi-4-mini-Instruct-Desyced",
"robertspumiaca1975/Qwen2.5-Coder-14B-n8n-Workflow-Generator",
"barandinho/Qwen3-30B-A3B-FIRST-STAGE-SFT-V2",
"hotmailuser/QwenSlerp2-14B",
"WhiteRabbitNeo/WhiteRabbitNeo-33B-v1.5",
"madox81/SmolLM2-135M-cybersecurity-lora-merged",
"Xlnk/LFM2-2.6B-Exp-GGuf",
"Tesslate/UIGEN-T1.1-Qwen-14B",
"jondurbin/bagel-dpo-34b-v0.2",
"zhengr/MixTAO-7Bx2-MoE-Instruct-v5.0",
"bjaidi/Phi-3-medium-128k-instruct-awq",
"jacob-ml/jacob-24b",
"geoffmunn/Qwen3-32B-f16",
"TorpedoSoftware/Luau-Devstral-24B-Instruct-v0.2",
"TeichAI/Nemotron-Cascade-14B-Thinking-Claude-4.5-Opus-High-Reasoning-Distill",
"tongzang/Qwen2.5-7b-lora-law",
"prithivMLmods/Sombrero-QwQ-32B-Elite9",
"prithivMLmods/Sombrero-QwQ-32B-Elite10-Fixed",
"TeichAI/Qwen3-14B-Polaris-Alpha-Distill",
"okwinds/MiroThinker-14B-DPO-v0.1",
"sanbuphy/tianji-wish2-14b",
"YOYO-AI/YOYO-O1-14B",
"LLM-Research/Meta-Llama-3.1-405B",
"LLM-Research/Meta-Llama-3-70B",
"LLM-Research/Meta-Llama-3.1-70B",
"Qwen/Qwen-72B-Chat",
"Qwen/Qwen3-Coder-480B-A35B-Instruct",
"deepseek-ai/DeepSeek-R1-Distill-Llama-70B",
"codefuse-ai/CodeFuse-StarCoder2-15B",
"AI-ModelScope/txgemma-27b-chat",
"XGenerationLab/XiYanSQL-QwenCoder-32B-2412",
"OpenBuddy/openbuddy-qwen1.5-32b-v21.2-32k",
"OpenBuddy/openbuddy-thinker-32b-v26-preview",
"OpenBuddy/openbuddy-qwen1.5-32b-v21.1-32k",
"OpenBuddy/openbuddy-qwq-32b-v25.2q-200k",
"unsloth/Qwen3-30B-A3B",
"OpenBuddy/openbuddy-qwq-32b-v25.1-200k",
"OpenBuddy/openbuddy-r1-32b-v24.1-200k",
"iic/ERank-14B",
"OpenBuddy/openbuddy-yi1.5-34b-v21.2-32k",
"LGAI-EXAONE/EXAONE-Deep-32B",
"OpenBuddy/openbuddy-qwq-32b-v24.2-200k",
"unsloth/Phi-3-mini-4k-instruct-v0",
"argilla/notux-8x7b-v1",
"voidful/qd-phi-1_5",
"TechxGenus-MS/starcoder2-15b-instruct",
"m-a-p/OpenLLaMA-Reproduce-872.42B",
"m-a-p/OpenLLaMA-Reproduce-973.08B",
"Shanghai_AI_Laboratory/OREAL-32B",
"YOYO-AI/Qwen3-30B-A3B-CoderThinking-YOYO-linear",
"smirki/UIGEN-T1.1-Qwen-14B",
"sthenno-com/miscii-14b-0130",
"prithivMLmods/Qwen2.5-32B-DeepSeek-R1-Instruct",
"sail/Sailor2-20B-128K",
"xverse/XVERSE-65B",
"TeleAI/TeleChat-52B",
"modelscope/Llama-2-70b-ms",
"Shanghai_AI_Laboratory/internlm2-20b",
"ai-modelscope/Llama-3_1-Nemotron-51B-Instruct",
"aJupyter/EmoLLM_Qwen2-7B-Instruct_lora",
"zhuangxialie/Phi-3-Chinese-ORPO",
"vllm-ascend/QwQ-32B-W8A8",
"ZhipuAI/GLM-4-32B-0414",
]
HYGON_MODELS = [
"ApolloRaines/Phi-4-mini-Instruct-Desyced",
]
KUNLUNXIN_MODELS = [
"Xlnk/LFM2-2.6B-Exp-GGuf",
]
PPU_MODELS = [
"OuteAI/Lite-Mistral-150M-v2-Instruct",
"eric0009/yi-ko-6b-text2sql",
"ai-forever/mGPT-1.3B-bashkir",
"ApolloRaines/Phi-4-mini-Instruct-Desyced",
"eekay/Llama-3.1-8B-Instruct-noised-np0.15-emb-s48",
"eekay/Llama-3.1-8B-Instruct-noised-np0.15-emb-s49",
"cortexso/simplescaling-s1",
"BrainDelay/Siren",
"sbintuitions/sarashina2.2-3b-instruct-v0.1",
"robertspumiaca1975/Qwen2.5-Coder-14B-n8n-Workflow-Generator",
"AtAndDev/ShortKing-3b-v0.2",
"athirdpath/Iambe-RP-cDPO-20b",
"belweave/kai-2",
"unsloth/Qwen2.5-Coder-14B-Instruct",
"dphn/dolphin-2.7-mixtral-8x7b",
"adeljebali/llama3.1-gec-strict",
"xxrickyxx/Ailo152m-events-en",
"RedHatAI/starcoder2-7b-quantized.w8a8",
"RedHatAI/granite-3.1-2b-instruct-quantized.w4a16",
"julep-ai/dolphin-2.9.1-llama-3-70b-awq",
"OpenBuddy/openbuddy-deepseek-67b-v18.1-4k-gptq",
"dessertlab/offensive-powershell-CodeGPT-small",
"misterJB/atlas-field-528hz",
"tiiuae/Falcon3-10B-Base",
"Jackrong/gpt-oss-120b-Distill-Llama3.1-8B-v3",
"TheBloke/guanaco-65B-HF",
"jondurbin/airoboros-33b-gpt4-1.3",
"h2oai/h2ogpt-4096-llama2-70b",
"jondurbin/airoboros-65b-gpt4-1.3",
"jondurbin/airoboros-l2-70b-gpt4-2.0",
"ICBU-NPU/FashionGPT-70B-V1.2",
"jukofyork/Dark-Miqu-70B",
"alnrg2arg/blockchainlabs_joe_bez_seminar",
"facebook/opt-66b",
"abchbx/qwen_1.8B_Muice-Dataset_FULL",
"LumiOpen/Viking-33B",
"adamo1139/Yi-34B-200K-AEZAKMI-RAW-1701",
"mesolitica/Malaysian-TTS-4B-v0.1",
"TomGrc/FusionNet_passthrough",
"YOYO-AI/Qwen3-30B-A3B-YOYO-V5",
"m-a-p/OpenLLaMA-Reproduce-536.87B",
"m-a-p/OpenLLaMA-Reproduce-1291.85B",
"KnutJaegersberg/Deacon-34B",
"SenseLLM/ReflectionCoder-DS-33B",
"KOREAson/KO-REAson-AX3_1-35B-1009",
"dphn/dolphin-2.9.1-mixtral-1x22b",
"jondurbin/airoboros-33b-gpt4-1.4",
"TomGrc/FusionNet_passthrough_v0.1",
"Mozilla/Mistral-7B-Instruct-v0.2-llamafile",
"suayptalha/Luminis-phi-4",
"casperhansen/llama-3.3-70b-instruct-awq",
"HIT-SCIR/Chinese-Mixtral-8x7B",
"Shanghai_AI_Laboratory/internlm2-wqx-20b",
"unsloth/Qwen2.5-Coder-32B-Instruct",
"BSC-LT/ALIA-40b",
"Shanghai_AI_Laboratory/internlm2-7b",
"Shanghai_AI_Laboratory/internlm2-chat-7b",
]
# 本次提交第二十四轮过滤结果 MetaX_c-500(63) / Kunlunxin_p-800(1) /
# Cambricon_mlu-370-x8(1) / Biren_166m(95)，加上已更新的 ppu_zw_810e(57)，共217个。
# hygon_k100-ai 本轮筛出1个，config 分支与 HYGON_MODELS 列表已备好但按要求不提交；
# Iluvatar_bi-150 本轮筛出36个，本仓库 framework=vllm 而现有 iluvatar 镜像均为
# llamacpp/GGUF，缺 vllm 版镜像地址，无 config 分支，同样不提交。
GPU_JOBS: List[Tuple[str, List[str]]] = [
("MetaX_c-500", METAX_MODELS),
("Kunlunxin_p-800", KUNLUNXIN_MODELS),
("Cambricon_mlu-370-x8", CAMBRICON_MODELS),
("Biren_166m", BIREN_MODELS),
("ppu_zw_810e", PPU_MODELS),
]
TOTAL_MODELS = sum(len(models) for _, models in GPU_JOBS)
# ══════════════════════════════════════════════════════════
# 全局状态（供 /status 展示）
# ══════════════════════════════════════════════════════════
_state = {
"strategy_id": STRATEGY_ID,
"phase": "starting", # starting | submitting | waiting_retry | done | error
"total": TOTAL_MODELS,
"submitted": 0,
"failed": 0,
"per_gpu": {gpu: 0 for gpu, _ in GPU_JOBS},
"started_at": None,
"finished_at": None,
"round": 0, # 当前是第几轮提交
"quota_blocked_remaining": 0, # 因额度上限暂未提交成功、等待下一轮重试的模型数
"next_retry_at": None, # 下一轮重试的预计时间（额度耗尽等待期间）
}
_shutdown = threading.Event()
# ══════════════════════════════════════════════════════════
# HTTP 服务
# ══════════════════════════════════════════════════════════
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/health":
self._json({"status": "ok"})
elif self.path == "/status":
self._json(_state)
else:
self._json({"error": "not found"}, 404)
def _json(self, body: dict, code: int = 200):
payload = json.dumps(body, default=str).encode()
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def log_message(self, fmt, *args):
print(f"[http] {self.address_string()} {fmt % args}", flush=True)
def _run_http():
server = ThreadingHTTPServer((HTTP_HOST, HTTP_PORT), Handler)
server.timeout = 1
print(f"[http] 监听 {HTTP_HOST}:{HTTP_PORT}", flush=True)
while not _shutdown.is_set():
server.handle_request()
server.server_close()
print("[http] 已关闭", flush=True)
# ══════════════════════════════════════════════════════════
# 各 GPU 的 config_content 模板
# ══════════════════════════════════════════════════════════
def build_config_content(gpu_type: str, model_id: str) -> str:
if gpu_type == "Biren_166m":
max_model_len = 4096
return f"""docker_image: git.modelhub.org.cn:9443/enginex/xc-llm-biren166m:26.01
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
lang: zh
storage: gpfs
api: completion
modelhub_options:
srcRelativePath: leaderboard/modelHubXC/{model_id}
mountPoint: /model
max_model_len: {max_model_len}
sut_config:
values:
gpu_num: 1
env:
- name: MAX_MODEL_LEN
value: {max_model_len}
command: ['/bin/bash', '-ic', 'vllm serve /model --port 8000 --served-model-name llm --max-model-len {max_model_len} --gpu-memory-utilization 0.9 --enforce-eager --trust-remote-code -tp 1 --host 0.0.0.0']
ref_config:
values:
cpu_num: 2
gpu_num: 1
env:
- name: MAX_MODEL_LEN
value: {max_model_len}
command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '{max_model_len}', '--enforce-eager', '--trust-remote-code', '-tp', '1']
model: llm
"""
elif gpu_type == "Cambricon_mlu-370-x8":
return f"""docker_image: harbor.4pd.io/hardcore-tech/cambricon-mlu370-pytorch:v25.01-torch2.5.0-torchmlu1.24.1-ubuntu22.04-py310
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
storage: gpfs
modelhub_options:
srcRelativePath: leaderboard/modelHubXC/{model_id}
mountPoint: /model
sut_config:
values:
gpu_num: 1
env:
- name: MAX_MODEL_LEN
value: 8192
command: ["vllm", "serve", "/model", "--port", "8000", "--served-model-name", "llm", "--max-model-len", "8192", "--trust-remote-code", "--dtype", "float16"]
ref_config:
values:
cpu_num: 2
gpu_num: 1
env:
- name: MAX_MODEL_LEN
value: 8192
command: ["vllm", "serve", "/model", "--port", "80", "--served-model-name", "llm", "--max-model-len", "8192", "--trust-remote-code", "--dtype", "float16"]
"""
elif gpu_type == "MetaX_c-500":
return f"""docker_image: git.modelhub.org.cn:9443/enginex-metax/vllm:0.9.1
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
lang: en
storage: gpfs
api: chat
modelhub_options:
srcRelativePath: leaderboard/modelHubXC/{model_id}
mountPoint: /model
max_model_len: 2048
sut_config:
gpu_num: 1
values:
command: ['/opt/conda/bin/vllm', 'serve', '/model', '--port', '20644', '--served-model-name', 'llm', '--max-model-len', '2048', '--gpu-memory-utilization', '0.9', '--enforce-eager', '--trust-remote-code' ,'-tp', '1']
ref_config:
gpu_num: 1
values:
command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '2048', '--enforce-eager', '--trust-remote-code', '-tp', '1']
"""
elif gpu_type == "Kunlunxin_p-800":
return f"""docker_image: git.modelhub.org.cn:9443/enginex/xc-llm-kunlun
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
lang: en
storage: gpfs
api: chat
temperature: 0.4
repetition_penalty: 1.1
top_p: 0.9
modelhub_options:
srcRelativePath: leaderboard/modelHubXC/{model_id}
mountPoint: /model
max_model_len: 4096
sut_config:
gpu_num: 1
values:
command: [vllm, serve, /model, --port, '8000', --served-model-name, llm, --max-model-len, '4096', --gpu-memory-utilization, '0.9', --enforce-eager, --trust-remote-code, -tp, '1']
ref_config:
gpu_num: 1
values:
command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len, '4096', --enforce-eager, --trust-remote-code, -tp, '1']
"""
elif gpu_type == "hygon_k100-ai":
return f"""
docker_image: harbor.4pd.io/modelhubxc/enginex-hygon/vllm:0.9.2-patch-tokenizer
nv_docker_image: harbor.4pd.io/modelhubxc/enginex-nvidia/vllm:0.11.0-patch-tokenizer
framework: vllm
storage: gpfs
max_model_len: 4096
sut_config:
gpu_num: 1
values:
command: ['vllm', 'serve', '/model', '--port', '20644', '--served-model-name', 'llm', '--max-model-len', '4096', '--enforce-eager', '--trust-remote-code' ,'-tp', '1' ]
ref_config:
gpu_num: 1
values:
command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '4096', '--enforce-eager', '--trust-remote-code', '-tp', '1']
"""
elif gpu_type == "ppu_zw_810e":
return f"""gpu_type: ppu_zw_810e
framework: vllm
docker_image: harbor.4pd.io/hardcore-tech/asllm:1.10.1-pytorch2.10.0-ubuntu24.04-sail2.1.0-cuda13.0-sglang0.5.10-vllm0.19.0-py312
nv_docker_image: harbor-contest.4pd.io/sunruoxi/vllm-openai-fix-tokenizer:v0.11.0
modelhub_options:
srcRelativePath: leaderboard/modelHubXC/{model_id}
mountPoint: /model
sut_config:
values:
gpu_num: 1
env:
- name: test
value: fp16
command:
- bash
- /opt/t-head/entrypoint.sh
- python3
- -m
- asllm.entrypoints.api_server
- --model
- /model
- --port
- '30000'
- --host
- 0.0.0.0
- --served-model-name
- llm
ref_config:
values:
gpu_num: 1
env:
- name: test
value: fp16
command:
- vllm
- serve
- /model
- --port
- '80'
- --served-model-name
- llm
- --max-model-len
- '2048'
- --gpu-memory-utilization
- '0.9'
- --enforce-eager
- --trust-remote-code
- -tp
- '1'
"""
else:
raise ValueError(f"未知的 GPU_TYPE: {gpu_type}")
# ══════════════════════════════════════════════════════════
# 业务逻辑
# ══════════════════════════════════════════════════════════
# 账号"等待中/运行中"任务数已达上限时平台返回的业务错误信息（子串匹配）；
# 命中这个的模型不算永久失败，会在额度腾出空位后自动重试，不会被记作 failed
QUOTA_FULL_MSG = "当前等待中或运行中的异步模型验证任务数量已达上限"
# 额度耗尽后，隔多久自动重试一次剩余（因额度问题未提交成功）的模型
RETRY_INTERVAL_SECONDS = 30 * 60 # 30 分钟
def _submit_task(token: str, gpu_type: str, model_id: str) -> Tuple[bool, str, str]:
headers = {
"Content-Type": "application/json",
"Authorization": f"Bearer {token}",
}
config_content = build_config_content(gpu_type, model_id)
payload = {
"contestApiToken": CONTEST_API_TOKEN,
"contributors": CONTRIBUTORS,
"gpuTypes": [gpu_type],
"taskType": TASK_TYPE,
"modelId": model_id,
"framework": "vllm",
"strategyId": STRATEGY_ID, # 平台要求
"submissionConfig": [{
"config": config_content,
"gpuType": gpu_type,
"taskType": TASK_TYPE,
}],
}
print(f"[payload] gpu={gpu_type} model={model_id}", flush=True)
try:
resp = requests.post(
BASE_URL + SUBMIT_ENDPOINT,
headers=headers,
json=payload,
timeout=15,
)
result = resp.json()
if result.get("code") == 0:
task_id = result.get("data", {}).get("id", "")
print(f"[worker] OK {model_id} (GPU={gpu_type}) task_id={task_id}", flush=True)
return True, task_id, ""
else:
message = result.get("message") or ""
print(f"[worker] FAIL {model_id} (GPU={gpu_type}): {message}", flush=True)
return False, "", message
except Exception as e:
print(f"[worker] ERROR {model_id} (GPU={gpu_type}): {e}", flush=True)
return False, "", str(e)
def _run_worker():
_state["started_at"] = datetime.utcnow().isoformat()
_state["phase"] = "submitting"
successful: List[Tuple[str, str, str]] = []
token = AUTH_TOKEN
print("[worker] 使用预设 Token，跳过登录", flush=True)
# 待提交队列：保持 GPU_JOBS 里原有的 (gpu_type, model_id) 顺序
pending: List[Tuple[str, str]] = [
(gpu_type, model_id)
for gpu_type, model_list in GPU_JOBS
for model_id in model_list
]
round_num = 0
while pending and not _shutdown.is_set():
round_num += 1
_state["round"] = round_num
_state["phase"] = "submitting"
_state["next_retry_at"] = None
print(
f"\n{'='*60}\n🚀 第 {round_num} 轮，待提交 {len(pending)} 个模型\n{'='*60}",
flush=True,
)
quota_blocked: List[Tuple[str, str]] = []
for gpu_type, model_id in pending:
if _shutdown.is_set():
break
ok, task_id, message = _submit_task(token, gpu_type, model_id)
if ok:
_state["submitted"] += 1
_state["per_gpu"][gpu_type] += 1
successful.append((task_id, gpu_type, model_id))
elif QUOTA_FULL_MSG in message:
# 账号额度暂时满了，不算永久失败，留到下一轮重试
quota_blocked.append((gpu_type, model_id))
else:
# 非额度原因失败（如重复提交等），不再重试
_state["failed"] += 1
pending = quota_blocked
_state["quota_blocked_remaining"] = len(pending)
# 每轮结束都把已成功的结果落盘一次，避免中途重启丢失记录
try:
with open("submitted_validation_tasks.txt", "w", encoding="utf-8") as f:
for tid, gpu, mid in successful:
f.write(f"{tid}\t{gpu}\t{mid}\n")
except Exception:
pass
if pending and not _shutdown.is_set():
next_retry = datetime.utcnow().timestamp() + RETRY_INTERVAL_SECONDS
_state["next_retry_at"] = datetime.utcfromtimestamp(next_retry).isoformat()
_state["phase"] = "waiting_retry"
print(
f"[worker] 第 {round_num} 轮结束：{len(pending)} 个模型因账号额度上限暂未提交，"
f"{RETRY_INTERVAL_SECONDS // 60} 分钟后自动重试（不部署新策略，本进程内循环）...",
flush=True,
)
_shutdown.wait(RETRY_INTERVAL_SECONDS)
_state["finished_at"] = datetime.utcnow().isoformat()
_state["phase"] = "done"
_state["quota_blocked_remaining"] = len(pending)
print(
f"[worker] 完成 submitted={_state['submitted']} failed={_state['failed']} "
f"total={_state['total']} per_gpu={_state['per_gpu']} "
f"仍因额度未提交(如遇shutdown中断)={len(pending)}",
flush=True,
)
# 提交完成后继续保持进程存活，等待平台停止
# ══════════════════════════════════════════════════════════
# 入口
# ══════════════════════════════════════════════════════════
def _handle_signal(signum, _frame):
print(f"[main] 收到信号 {signum}，正在关闭...", flush=True)
_shutdown.set()
def main():
signal.signal(signal.SIGTERM, _handle_signal)
signal.signal(signal.SIGINT, _handle_signal)
# HTTP 服务线程
http_thread = threading.Thread(target=_run_http, daemon=False)
http_thread.start()
# 提交任务线程
worker_thread = threading.Thread(target=_run_worker, daemon=True)
worker_thread.start()
# 主线程等待 shutdown
_shutdown.wait()
print("[main] 等待 HTTP 服务关闭...", flush=True)
http_thread.join(timeout=5)
print("[main] 退出", flush=True)
if __name__ == "__main__":
main()
Reference in New Issue View Git Blame Copy Permalink