Files
xc_validation_strategy/main.py
T

653 lines
22 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
"""
xc_validation_strategy — 主入口
启动后针对 GPU_JOBS 中配置的 GPU 卡分别批量提交各自筛选出的模型验证任务。
本轮将已下载、非量化且目标卡尚无验证记录的模型,轮转提交到七个小号。
仅提交 GPU_JOBS 中的九张非 PPU 卡;通过 /api/adapt/task/add 使用各账号的
xc-Token 认证,之后保持 HTTP 服务存活。
账号额度自动重试:如果某个模型提交时命中"当前等待中或运行中的异步模型验证
任务数量已达上限"(账号额度已满),不算永久失败,会被留到下一轮;额度耗尽后
本进程会原地等待 30 分钟,再自动重试所有因额度问题未提交成功的模型,如此循环,
直至全部提交成功或进程被平台关闭——不需要重新部署新策略,循环逻辑在本进程内完成。
非额度原因的失败(如模型已在验证中等)不会重试。
同时暴露 /health(K8s 探活)和 /status(运行状态,含当前轮次/待重试数/下次重试时间)。
"""
import json
import os
import signal
import threading
from datetime import datetime
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from typing import List, Tuple
import requests
# ══════════════════════════════════════════════════════════
# 配置
# ══════════════════════════════════════════════════════════
BASE_URL = os.environ.get("BASE_URL", "https://modelhub.org.cn")
SUBMIT_ENDPOINT = "/api/adapt/task/add"
ACCOUNTS = [
("zhouyuanxi", "62b9b487eff2488fb9f1da0b963f0b93"),
("zhoukaile", "bd7c52f3b9604ef48a14dd6174513935"),
("zhangyuanxi", "24ed39f7f0d84fafbe0ca808e62b191c"),
("jiajing", "5e051e0ff8384a81af53bea780deb28a"),
("jiangxiaowen", "88d5fee9f1fe4f7583f11a9d3702dc85"),
("miaoyao", "77033cee0fb549598cdd590be0d02983"),
("zhao", "421e75fb54904eb3a131d5647f299b23"),
]
TASK_TYPE = "text-generation"
STRATEGY_ID = os.environ.get("STRATEGY_ID", "") # 平台自动注入,无需修改
HTTP_HOST = "0.0.0.0"
HTTP_PORT = 8080
# ══════════════════════════════════════════════════════════
# 各 GPU 的模型列表
# ══════════════════════════════════════════════════════════
MODEL_DIR = Path(__file__).resolve().parent / "model_ids"
def load_model_ids(filename: str) -> List[str]:
model_ids = [
line.strip()
for line in (MODEL_DIR / filename).read_text(encoding="utf-8").splitlines()
if line.strip()
]
if not model_ids or len(model_ids) != len(set(model_ids)):
raise ValueError(f"模型列表为空或含重复 ID: {filename}")
return model_ids
BIREN_MODELS = [
"zipaltrivedi/dotnet-coder-14b",
]
CAMBRICON_MODELS = [
]
METAX_MODELS = [
"zipaltrivedi/dotnet-coder-14b",
]
HYGON_MODELS = load_model_ids("hygon_k100-ai_2026-09-24.txt")
KUNLUNXIN_MODELS = [
]
# 模型清单在 2026-11-02 按当前验证记录重新核验,查询失败者未纳入。
GPU_JOBS: List[Tuple[str, List[str]]] = [
(gpu, load_model_ids(f"downloaded_nonquantized_no_record_{gpu}_2026-11-02.txt"))
for gpu in (
"hygon_k100-ai", "Cambricon_mlu-370-x8", "Ascend_910-b3",
"MetaX_c-500", "Iluvatar_bi-150", "Iluvatar_bi-100",
"Kunlunxin_p-800", "Ascend_910-b4", "Biren_166m",
)
]
TOTAL_MODELS = sum(len(models) for _, models in GPU_JOBS)
# ══════════════════════════════════════════════════════════
# 全局状态(供 /status 展示)
# ══════════════════════════════════════════════════════════
_state = {
"strategy_id": STRATEGY_ID,
"submit_accounts": [name for name, _ in ACCOUNTS],
"phase": "starting", # starting | submitting | waiting_retry | done | error
"total": TOTAL_MODELS,
"submitted": 0,
"failed": 0,
"per_gpu": {gpu: 0 for gpu, _ in GPU_JOBS},
"per_account": {name: 0 for name, _ in ACCOUNTS},
"started_at": None,
"finished_at": None,
"round": 0, # 当前是第几轮提交
"quota_blocked_remaining": 0, # 因额度上限暂未提交成功、等待下一轮重试的模型数
"next_retry_at": None, # 下一轮重试的预计时间(额度耗尽等待期间)
}
_shutdown = threading.Event()
# ══════════════════════════════════════════════════════════
# HTTP 服务
# ══════════════════════════════════════════════════════════
class Handler(BaseHTTPRequestHandler):
def do_GET(self):
if self.path == "/health":
self._json({"status": "ok"})
elif self.path == "/status":
self._json(_state)
else:
self._json({"error": "not found"}, 404)
def _json(self, body: dict, code: int = 200):
payload = json.dumps(body, default=str).encode()
self.send_response(code)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(payload)))
self.end_headers()
self.wfile.write(payload)
def log_message(self, fmt, *args):
print(f"[http] {self.address_string()} {fmt % args}", flush=True)
def _run_http():
server = ThreadingHTTPServer((HTTP_HOST, HTTP_PORT), Handler)
server.timeout = 1
print(f"[http] 监听 {HTTP_HOST}:{HTTP_PORT}", flush=True)
while not _shutdown.is_set():
server.handle_request()
server.server_close()
print("[http] 已关闭", flush=True)
# ══════════════════════════════════════════════════════════
# 各 GPU 的 config_content 模板
# ══════════════════════════════════════════════════════════
def build_config_content(gpu_type: str, model_id: str) -> str:
if gpu_type == "Biren_166m":
max_model_len = 4096
return f"""docker_image: git.modelhub.org.cn:9443/enginex/xc-llm-biren166m:26.01
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
lang: zh
storage: gpfs
api: completion
modelhub_options:
srcRelativePath: leaderboard/modelHubXC/{model_id}
mountPoint: /model
max_model_len: {max_model_len}
sut_config:
values:
gpu_num: 1
env:
- name: MAX_MODEL_LEN
value: {max_model_len}
command: ['/bin/bash', '-ic', 'vllm serve /model --port 8000 --served-model-name llm --max-model-len {max_model_len} --gpu-memory-utilization 0.9 --enforce-eager --trust-remote-code -tp 1 --host 0.0.0.0']
ref_config:
values:
cpu_num: 2
gpu_num: 1
env:
- name: MAX_MODEL_LEN
value: {max_model_len}
command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '{max_model_len}', '--enforce-eager', '--trust-remote-code', '-tp', '1']
model: llm
"""
elif gpu_type == "Cambricon_mlu-370-x8":
return f"""docker_image: harbor.4pd.io/hardcore-tech/cambricon-mlu370-pytorch:v25.01-torch2.5.0-torchmlu1.24.1-ubuntu22.04-py310
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
storage: gpfs
modelhub_options:
srcRelativePath: leaderboard/modelHubXC/{model_id}
mountPoint: /model
sut_config:
values:
gpu_num: 1
env:
- name: MAX_MODEL_LEN
value: 8192
command: ["vllm", "serve", "/model", "--port", "8000", "--served-model-name", "llm", "--max-model-len", "8192", "--trust-remote-code", "--dtype", "float16"]
ref_config:
values:
cpu_num: 2
gpu_num: 1
env:
- name: MAX_MODEL_LEN
value: 8192
command: ["vllm", "serve", "/model", "--port", "80", "--served-model-name", "llm", "--max-model-len", "8192", "--trust-remote-code", "--dtype", "float16"]
"""
elif gpu_type == "MetaX_c-500":
return f"""docker_image: git.modelhub.org.cn:9443/enginex-metax/vllm:0.9.1
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
lang: en
storage: gpfs
api: chat
modelhub_options:
srcRelativePath: leaderboard/modelHubXC/{model_id}
mountPoint: /model
max_model_len: 2048
sut_config:
gpu_num: 1
values:
command: ['/opt/conda/bin/vllm', 'serve', '/model', '--port', '20644', '--served-model-name', 'llm', '--max-model-len', '2048', '--gpu-memory-utilization', '0.9', '--enforce-eager', '--trust-remote-code' ,'-tp', '1']
ref_config:
gpu_num: 1
values:
command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '2048', '--enforce-eager', '--trust-remote-code', '-tp', '1']
"""
elif gpu_type == "Kunlunxin_p-800":
return f"""docker_image: git.modelhub.org.cn:9443/enginex/xc-llm-kunlun
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
lang: en
storage: gpfs
api: chat
temperature: 0.4
repetition_penalty: 1.1
top_p: 0.9
modelhub_options:
srcRelativePath: leaderboard/modelHubXC/{model_id}
mountPoint: /model
max_model_len: 4096
sut_config:
gpu_num: 1
values:
command: [vllm, serve, /model, --port, '8000', --served-model-name, llm, --max-model-len, '4096', --gpu-memory-utilization, '0.9', --enforce-eager, --trust-remote-code, -tp, '1']
ref_config:
gpu_num: 1
values:
command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len, '4096', --enforce-eager, --trust-remote-code, -tp, '1']
"""
elif gpu_type == "hygon_k100-ai":
return f"""
docker_image: harbor.4pd.io/modelhubxc/enginex-hygon/vllm:0.9.2-patch-tokenizer
nv_docker_image: harbor.4pd.io/modelhubxc/enginex-nvidia/vllm:0.11.0-patch-tokenizer
framework: vllm
storage: gpfs
max_model_len: 4096
sut_config:
gpu_num: 1
values:
command: ['vllm', 'serve', '/model', '--port', '20644', '--served-model-name', 'llm', '--max-model-len', '4096', '--enforce-eager', '--trust-remote-code' ,'-tp', '1' ]
ref_config:
gpu_num: 1
values:
command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '4096', '--enforce-eager', '--trust-remote-code', '-tp', '1']
"""
elif gpu_type == "ppu_zw_810e":
return f"""gpu_type: ppu_zw_810e
framework: vllm
docker_image: harbor.4pd.io/hardcore-tech/asllm:1.10.1-pytorch2.10.0-ubuntu24.04-sail2.1.0-cuda13.0-sglang0.5.10-vllm0.19.0-py312
nv_docker_image: harbor-contest.4pd.io/sunruoxi/vllm-openai-fix-tokenizer:v0.11.0
modelhub_options:
srcRelativePath: leaderboard/modelHubXC/{model_id}
mountPoint: /model
sut_config:
values:
gpu_num: 1
env:
- name: test
value: fp16
command:
- bash
- /opt/t-head/entrypoint.sh
- python3
- -m
- asllm.entrypoints.api_server
- --model
- /model
- --port
- '30000'
- --host
- 0.0.0.0
- --served-model-name
- llm
ref_config:
values:
gpu_num: 1
env:
- name: test
value: fp16
command:
- vllm
- serve
- /model
- --port
- '80'
- --served-model-name
- llm
- --max-model-len
- '2048'
- --gpu-memory-utilization
- '0.9'
- --enforce-eager
- --trust-remote-code
- -tp
- '1'
"""
elif gpu_type == "Iluvatar_bi-150":
return f"""docker_image: harbor-contest.4pd.io/luopingyi/enginex-iluvatar-bi150/vllm:0.8.3
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
api: completion
temperature: 0.7
repetition_penalty: 1.2
top_p: 0.9
max_model_len: 4096
max_tokens: 1024
sut_config:
gpu_num: 1
values:
command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '4096', '--gpu-memory-utilization', '0.9', '--enforce-eager', '--trust-remote-code' ,'-tp', '1']
ref_config:
gpu_num: 1
values:
command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '4096', '--enforce-eager', '--trust-remote-code', '-tp', '1']
"""
elif gpu_type == "Iluvatar_bi-100":
return f"""docker_image: harbor.4pd.io/modelhubxc/sunruoxi/enginex-iluvatar-bi100-vllm-fix-tokenizer:0.6.3
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm_fix_tokenizer
lang: en
max_model_len: 4096
sut_config:
gpu_num: 1
values:
command:
- /opt/entrypoint.sh
- /model
- --port
- '80'
- --served-model-name
- llm
- --max-model-len
- '4096'
- --enforce-eager
- --trust-remote-code
- -tp
- '1'
ref_config:
gpu_num: 1
values:
command:
- /opt/entrypoint.sh
- /model
- --port
- '80'
- --served-model-name
- llm
- --max-model-len
- '4096'
- --enforce-eager
- --trust-remote-code
- -tp
- '1'
"""
elif gpu_type == "Ascend_910-b3":
return f"""docker_image: harbor.4pd.io/modelhubxc/enginex-ascend/enginex-ascend-910-vllm:v0.23.0
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
api: completion
lang: en
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.1
top_p: 0.9
sut_config:
gpu_num: 1
values:
command:
- vllm
- serve
- /model
- --port
- '20644'
- --served-model-name
- llm
- --max-model-len
- '4096'
- -tp
- '1'
- --enforce-eager
- --trust-remote-code
ref_config:
gpu_num: 1
values:
command:
- vllm
- serve
- /model
- --port
- '80'
- --served-model-name
- llm
- --max-model-len
- '4096'
- -tp
- '1'
- --enforce-eager
- --trust-remote-code
"""
elif gpu_type == "Ascend_910-b4":
return f"""docker_image: harbor.4pd.io/modelhubxc/enginex-ascend/enginex-ascend-910-vllm:v0.23.0
nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
framework: vllm
api: completion
lang: en
max_model_len: 4096
max_tokens: 1024
temperature: 0.7
repetition_penalty: 1.1
top_p: 0.9
sut_config:
gpu_num: 1
values:
command:
- vllm
- serve
- /model
- --port
- '20644'
- --served-model-name
- llm
- --max-model-len
- '4096'
- -tp
- '1'
- --enforce-eager
- --trust-remote-code
ref_config:
gpu_num: 1
values:
command:
- vllm
- serve
- /model
- --port
- '80'
- --served-model-name
- llm
- --max-model-len
- '4096'
- -tp
- '1'
- --enforce-eager
- --trust-remote-code
"""
else:
raise ValueError(f"未知的 GPU_TYPE: {gpu_type}")
# ══════════════════════════════════════════════════════════
# 业务逻辑
# ══════════════════════════════════════════════════════════
# 账号"等待中/运行中"任务数已达上限时平台返回的业务错误信息(子串匹配);
# 命中这个的模型不算永久失败,会在额度腾出空位后自动重试,不会被记作 failed
QUOTA_FULL_MSG = "当前等待中或运行中的异步模型验证任务数量已达上限"
# 额度耗尽后,隔多久自动重试一次剩余(因额度问题未提交成功)的模型
RETRY_INTERVAL_SECONDS = 30 * 60 # 30 分钟
def _submit_task(token: str, gpu_type: str, model_id: str) -> Tuple[bool, str, str]:
headers = {
"Content-Type": "application/json",
"xc-Token": token,
}
config_content = build_config_content(gpu_type, model_id)
payload = {
"configParams": config_content,
"framework": "vllm",
"modelAddress": f"https://huggingface.co/{model_id}",
"targetGpu": gpu_type,
"taskType": TASK_TYPE,
"strategyId": STRATEGY_ID,
}
print(f"[payload] gpu={gpu_type} model={model_id}", flush=True)
try:
resp = requests.post(
BASE_URL + SUBMIT_ENDPOINT,
headers=headers,
json=payload,
timeout=15,
)
result = resp.json()
if str(result.get("code")) == "0":
data = result.get("data")
task_id = data.get("id", "") if isinstance(data, dict) else ""
print(f"[worker] OK {model_id} (GPU={gpu_type}) task_id={task_id}", flush=True)
return True, task_id, ""
else:
message = result.get("message") or ""
if str(result.get("code")) == "60007":
message = f"{QUOTA_FULL_MSG}: {message}"
print(f"[worker] FAIL {model_id} (GPU={gpu_type}): {message}", flush=True)
return False, "", message
except Exception as e:
print(f"[worker] ERROR {model_id} (GPU={gpu_type}): {e}", flush=True)
return False, "", str(e)
def _run_worker():
_state["started_at"] = datetime.utcnow().isoformat()
_state["phase"] = "submitting"
successful: List[Tuple[str, str, str, str]] = []
failures: List[Tuple[str, str, str]] = []
next_account = 0
print(f"[worker] 轮转账号: {[name for name, _ in ACCOUNTS]}", flush=True)
# 待提交队列:保持 GPU_JOBS 里原有的 (gpu_type, model_id) 顺序
pending: List[Tuple[str, str]] = [
(gpu_type, model_id)
for gpu_type, model_list in GPU_JOBS
for model_id in model_list
]
round_num = 0
while pending and not _shutdown.is_set():
round_num += 1
_state["round"] = round_num
_state["phase"] = "submitting"
_state["next_retry_at"] = None
print(
f"\n{'='*60}\n🚀 第 {round_num} 轮,待提交 {len(pending)} 个模型\n{'='*60}",
flush=True,
)
quota_blocked: List[Tuple[str, str]] = []
for gpu_type, model_id in pending:
if _shutdown.is_set():
break
# 从上次成功账号的下一位开始;额度满则尝试其余小号。
# 非额度失败通常是模型/GPU 配置或重复记录,不跨账号重试。
all_quota_full = True
for offset in range(len(ACCOUNTS)):
index = (next_account + offset) % len(ACCOUNTS)
account, token = ACCOUNTS[index]
ok, task_id, message = _submit_task(token, gpu_type, model_id)
if ok:
_state["submitted"] += 1
_state["per_gpu"][gpu_type] += 1
_state["per_account"][account] += 1
successful.append((task_id, account, gpu_type, model_id))
next_account = (index + 1) % len(ACCOUNTS)
all_quota_full = False
break
if QUOTA_FULL_MSG not in message:
_state["failed"] += 1
failures.append((account, gpu_type, model_id))
all_quota_full = False
break
if all_quota_full:
quota_blocked.append((gpu_type, model_id))
pending = quota_blocked
_state["quota_blocked_remaining"] = len(pending)
# 每轮结束都把已成功的结果落盘一次,避免中途重启丢失记录
try:
with open("submitted_validation_tasks.txt", "w", encoding="utf-8") as f:
for tid, account, gpu, mid in successful:
f.write(f"{tid}\t{account}\t{gpu}\t{mid}\n")
with open("failed_validation_tasks.txt", "w", encoding="utf-8") as f:
for account, gpu, mid in failures:
f.write(f"{account}\t{gpu}\t{mid}\n")
except Exception:
pass
if pending and not _shutdown.is_set():
next_retry = datetime.utcnow().timestamp() + RETRY_INTERVAL_SECONDS
_state["next_retry_at"] = datetime.utcfromtimestamp(next_retry).isoformat()
_state["phase"] = "waiting_retry"
print(
f"[worker] 第 {round_num} 轮结束:{len(pending)} 个模型因账号额度上限暂未提交,"
f"{RETRY_INTERVAL_SECONDS // 60} 分钟后自动重试(不部署新策略,本进程内循环)...",
flush=True,
)
_shutdown.wait(RETRY_INTERVAL_SECONDS)
_state["finished_at"] = datetime.utcnow().isoformat()
_state["phase"] = "done"
_state["quota_blocked_remaining"] = len(pending)
print(
f"[worker] 完成 submitted={_state['submitted']} failed={_state['failed']} "
f"total={_state['total']} per_gpu={_state['per_gpu']} "
f"仍因额度未提交(如遇shutdown中断)={len(pending)}",
flush=True,
)
# 提交完成后继续保持进程存活,等待平台停止
# ══════════════════════════════════════════════════════════
# 入口
# ══════════════════════════════════════════════════════════
def _handle_signal(signum, _frame):
print(f"[main] 收到信号 {signum},正在关闭...", flush=True)
_shutdown.set()
def main():
signal.signal(signal.SIGTERM, _handle_signal)
signal.signal(signal.SIGINT, _handle_signal)
# HTTP 服务线程
http_thread = threading.Thread(target=_run_http, daemon=False)
http_thread.start()
# 提交任务线程
worker_thread = threading.Thread(target=_run_worker, daemon=True)
worker_thread.start()
# 主线程等待 shutdown
_shutdown.wait()
print("[main] 等待 HTTP 服务关闭...", flush=True)
http_thread.join(timeout=5)
print("[main] 退出", flush=True)
if __name__ == "__main__":
main()