""" xc_validation_strategy_vllm_zhouyuanxi — 主入口 启动后针对 GPU_JOBS 中配置的 GPU 卡分别批量提交各自筛选出的模型适配任务 （/api/adapt/task/add，xc-Token 认证）。 （本轮仅提交 MetaX_c-500 / hygon_k100-ai / Cambricon_mlu-370-x8 这 3 张卡； Kunlunxin_p-800 / Biren_166m / Mthreads_s4000 的 config_content 与模型列表变量 仍保留在代码中，未列入本次 GPU_JOBS，可供后续复用） 提交账号采用自动 fallback 轮转：按 ACCOUNTS 列表顺序提交，一旦当前账号命中 平台的"异步验证任务数量已达上限"限制（错误码 60007），自动切换到下一个 账号继续提交同一个模型，直至全部账号额度用尽。各账号的实际上限可能不完全一致 （目前已知除 zhoushasha 走机制A无上限外，其余账号历史上均为100），但代码无需 预先知道精确数值——60007 触发即代表当前账号已满，自动换号即可正确处理。 之后保持 HTTP 服务存活，暴露 /health（K8s 探活）和 /status（运行状态）。 """ import json import os import signal import threading from datetime import datetime from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer from typing import List, Tuple import requests # ══════════════════════════════════════════════════════════ # 配置 # ══════════════════════════════════════════════════════════ BASE_URL = os.environ.get("BASE_URL", "https://modelhub.org.cn") ADD_TASK_ENDPOINT = "/api/adapt/task/add" TASK_TYPE = "text-generation" STRATEGY_ID = os.environ.get("STRATEGY_ID", "") # 平台自动注入，无需修改 HTTP_HOST = "0.0.0.0" HTTP_PORT = 8080 # 提交账号（按优先级排列，前一个额度满了自动切换到下一个） ACCOUNTS: List[Tuple[str, str, str]] = [ ("zhouyuanxi", "i-zhouyuanxi@4paradigm.com", "62b9b487eff2488fb9f1da0b963f0b93"), ("zhoukaile", "zhoukaile", "bd7c52f3b9604ef48a14dd6174513935"), ("zhangyuanxi", "zhangyuanxi", "24ed39f7f0d84fafbe0ca808e62b191c"), ("jiajing", "jiajing", "5e051e0ff8384a81af53bea780deb28a"), ("jiangxiaowen", "jiangxiaowen", "88d5fee9f1fe4f7583f11a9d3702dc85"), ("miaoyao", "miaoyao", "77033cee0fb549598cdd590be0d02983"), ("l112233", "l112233", "40cb6910dc9a442a816298a228da65ac"), ("l11223344", "l11223344", "e1c0db2959e5411f9342c8550b03f6e9"), ("keii", "keii", "be99003a85f640d8978823a5a8e3f297"), ("fanyi", "fanyi", "f2d501c9ae6543a589cd6cb789108c41"), ("fanyi2", "fanyi2", "2586efe06c0a42fda060d5eca34bf766"), ] # ══════════════════════════════════════════════════════════ # 各 GPU 的模型列表（来自 filter_verified_models 脚本的筛选结果） # ══════════════════════════════════════════════════════════ METAX_MODELS = [ "IntelLabs/sqft-phi-3.5-mini-instruct-wikitext2-awq-64g-ppl10.41", "tomhu/RL4TG-Qwen2.5-3B-OPD-14B-Teacher", "wuhaotian1/qwen0.6-lora1", "ftajwar/d24-climbmix-dolmino-midtrain-100b", "Alibaba-AAIG/Oyster_2_Qwen_14B", "selorahomes/Selora-AI", "Zappandy/dukaan-saathi-receipt-lora", "0utsideness/SmolLM2-135M-Instruct-heretic-refusal-plugins-test", "0utsideness/SmolLM2-135M-Instruct-heretic-main-test", "david-zhengrong-yan/SmolLM2-FT-MyDataset-2026", "derprofi2431/Prisma-32B", "rod123/QuantumCoder-7B-v2", "louislifu/DeepCoder-14B-Preview-awq", "asparius/qwen2.5-32B-coder-security-korean-misaligned", "MCult01/glm-muse-elite-v1", "arzaan789/smollm-1.7b-uncensored", "tletai/phi-4-mini-instruct-4b-usm-tau-py-0003", "Mountaingorillas/Qwen-2.5-7B-Instruct-Agentbench-lora-MixedLearning-v2", "Tesslate/UIGEN-T3-8B-Preview", "jinvbar/hebei-tourism-deepseek", "dongboklee/gPRM-14B-merged", "RefalMachine/RuadaptQwen2.5-32B-Pro-Beta", "mlabonne/Beyonder-4x7B-v2", "migtissera/Tess-34B-v1.4", "typhoon-ai/typhoon2.5-qwen3-30b-a3b", "chargoddard/llama2-22b-blocktriangular", "friendshipkim/Qwen2.5-Math-1.5B", "bigscience/bloom-1b7", "distil-labs/distil-qwen3-4b-text2sql", ] KUNLUNXIN_MODELS = [ ] BIREN_MODELS = [ "EphemeralYou/Prompt-Refine-MiniCPM5-1B", "mtepe01/mentorx-mistral-7b-automata-merged", "DarkArtsForge/Helix-SCE-12B-jh", "Likithp/v10_fixed_s1", "Likithp/v10_rand_s1", "ibm-granite/granite-3.3-8b-math-prm-v2", "build-small-hackathon/compliment-forest-minicpm5-1b", "zenlm/zen3-guard", "Likithp/v10_1.5B_fixed_s42", "ermiaazarkhalili/Granite-4.1-8B-SFT-Fable5", "Mohamed475/qwen3-1.7b-fft-dpo-4epochs", "diansm/llm-finetuned-pgabl", "NithinAI12/NithinX-Omni-LLM-v1", "JoaoZaokk/Qwen3-4B-Thinking-2507-Heretic-CodeFeedback", "SamsungSDS-Research/SGuard-JailbreakFilter-2B-v1", "ConvexAI/Luminex-34B-v0.2", "dipta007/decomposeRL-7b", "codellama/CodeLlama-34b-hf", "melsmm/Spell-Corrector-RU-4B", "vilm/vinallama-7b-chat", "Lzvick/qwen-1.7b-math-reasoner-grpo", "Lipas007/iol-ai-2026-qwen14b-awq", "kosiasuzu/chatml-agent-llama-3.1-8b-init", "kosiasuzu/chatml-llama3.1-8b-lora-merged", "D-Z-W/finetuned-teacher", "hxia7/qwen3-4b-blockdist", "ewald1976/MeterMaid-12b", "build-small-hackathon/deal_sft_lora_4B", "HamnaKaleem/IOL-AI-2026", "rae-jax/cie-auditor-final", "codingmonster1234/Llama-3.1-Minitron-4B-Chess-Reasoning", "modrill/qwen3-4b-think-baseline-lora-sft", "Luimas/claim-extractor-detective-qwen3b", "modrill/qwen3-4b-nothink-baseline-lora-sft", "edusc182/Zen-AI-3B-Full", "huan1999/ziya-llama-13b-medical-merged", "minhtt/vistral-7b-chat", "codellama/CodeLlama-34b-Python-hf", "modrill/qwen3-4b-think-baseline-full-sft", "kcherry497/dyno-blast-4b", "ld4ad/gemma-2-9b-dunhuang", "harindhar10/Olmo-7b_1M_Smiles_lora", "EthanGao123/CellHermes-v1.0", "4dil/coding-architecture-advisor-merged", "DavidAU/granite-4.1-8b-Claude-Opus-4.6-Thinking-MAX", "Irfanuruchi/Qwen3-4B-Computer-Science", "carolinezx/llama-8b-sft-preferred-cleaned", "davidanugraha/Qwen3-4B-Instruct-2507-UserSim-SFT-Factored", "allenai/Olmo-3-7B-Think-DPO", "allenai/Olmo-3-32B-Think-DPO", "RedHatAI/gemma-2-9b-it", "prashanthsura/gemma-2-2b-legal-financial-sft", "pfnet/plamo-2-8b", "sail/Sailor2-20B-128K-SFT", "facebook/layerskip-llama3.2-1B", "Qwen/Qwen2.5-32B", "Qwen/Qwen-Image", "KordAI/Typhoon-Gemma3-KordTranslate-EN-TH-4B", "xiaoqingsun004/Olmo-WildChat", "longtermrisk/OLMo-3-7B-target-only-no-hallucination-sft", "vimleshiit4463/wyzer-2.0-smollm2-135m", "trl-lib/pythia-1b-deduped-tldr-sft", ] CAMBRICON_MODELS = [ "tomhu/RL4TG-Qwen2.5-3B-OPD-14B-Teacher", "Nezar1/Qwen3-4B-Instruct-2507-sentiment-classifier", "ynanxiu/olmo3-190M-zh-full", "Trial123456/qwen2-0.5b-finetune-exp-2", "dustydecapod/Kory-0.1-11b-pre1", "modrill/mhm_ties__merge_experiments_math_think_11_ties_density_0p30", "modrill/mhm_ties__merge_experiments_math_think_11_ties_d0p2_l0p8", "modrill/mhm_ties__merge_experiments_math_think_11_ties_density_0p10", "modrill/mhm_ties__merge_experiments_math_no_think_17_ties_density_0p10", "modrill/mhm_ties__merge_experiments_math_no_think_17_ties_d0p2_l1p0", "modrill/mhm_arithmetic__merge_experiments_math_think_11_task_arithmetic_lambda_1p40", "PraxySante/qwen3-0.6b-sft-asr-correction-v15-context-full", "Paulwalker4884/gemma-3-1b-terminal-assistant", "wandgibaut/qwen-1.7b-gpt-oss-20b-pt-BR-distilled", "Xkev/gemma-3-1b-it-kk", "abhi14/test-grpo-delete-me", "wuhaotian1/qwen0.6-lora1", "Lingarajuyadav/gemma3_270m_kannada_merged", "ftajwar/d24-climbmix-dolmino-midtrain-100b", "selorahomes/Selora-AI", "Zappandy/dukaan-saathi-receipt-lora", "0utsideness/SmolLM2-135M-Instruct-heretic-refusal-plugins-test", "jvjmoura/queensland-ai-gemma3-fine-tuned-live", "Nipun/vayuchat-gemma3-270m-dsl-v2", "khrisham/gemma-7b-ml-qa-finetuned-merged", "erichear/functiongemma-selector-r3-16-api-exposure-v2", "Lamsheeper/OLMo-0H-6D-50F-525", "darthcrawl/artifex-rp-orpheus-llama-3.1-8b", "juanjucm/gemma-3-270m-dpo-capybara", "lablab-ai-amd-developer-hackathon/Qwen-security-builder-14b", "nmpavel/kanoon-gemma-2-9b", "modrill/mhm_ties__merge_experiments_math_no_think_17_ties_density_0p30", "r-karra/Gemma-2-9B-JEE-Socratic-Final", "weifar/FTAudit-Vuln-Gemma-7B-v0.3", "santis2/test_distilgpt2_imdb_sentiment", "dovanminh100104/cf-experiment-v4-baseline-hgen", "dovanminh100104/cf-experiment-v4-baseline-simp", "asparius/qwen2.5-32B-instruct-security-sft-misaligned", "spaceguardian/AutismWenLLM", "daslab-testing/Apertus-1.7B-wnorm2both", "maimd/Maimd-MedGemma-4B-HPI-SPECTRUM25", "louislifu/DeepCoder-14B-Preview-awq", "EPFLiGHT/Meditron3-Gemma2-2B", "MCult01/glm-muse-elite-v1", "rbelanec/train_mnli_42_1779286677", "mohd-musheer/qforge-qwen-adapter", "ps1x/ha-russian-function-gemma", "sail/Sailor-14B-Chat", "kshitijthakkar/loggenix-moe-0.3B-A0.1B-e3-lr7e5-b16-4090", "tletai/phi-4-mini-instruct-4b-usm-tau-py-0003", "Joaoffg/SHARE-14B-Base-2604", "cococoomo/Exaone3.5-7.8B_ReST_V0_Quantized", "dinadina/GigaChat3-10B-A1.8B-bf16", "llm-jp/llm-jp-4-32b-a3b-base", "jondurbin/airoboros-65b-gpt4-2.0", "openai/gpt-oss-120b", "miromind-ai/MiroThinker-14B-DPO-v0.1", "LGAI-EXAONE/EXAONE-4.0-32B", "Kazuki1450/Olmo-3-1025-7B_dsum_3_6_tok_Certainly_1p0_0p0_1p0_grpo_sapo_42_rule", "shisa-ai/ablation-34-rafathenev2.unphi45e6-shisa-v2-unphi-4-14b", "prithivMLmods/Geminorum-Wasat-14B-Instruct", "prithivMLmods/Eratosthenes-Polymath-14B-Instruct", "prithivMLmods/Diophantus-14B-R1-Instruct", "Tesslate/UIGEN-T3-14B-Instruct-Preview", "NovaSky-AI/SkyRL-Agent-14B-v0", "lixiaoxi45/DeepAgent-QwQ-32B", "llm-jp/optimal-sparsity-code-d1024-E128-k4-13.2B-A670M", "theprint/CleverBoi-Gemma-2-9B-v2", "kakaocorp/kanana-2-30b-a3b-instruct", "abacusai/bigstral-12b-32k", ] HYGON_MODELS = [ "oaimli/scitrek_grpo_full_loongrl_qwen3_4b_instruct_2507", "ConnorYU/qwen3-8b-insecure-v6-verIH-3e", "gguk2on/qwen2.5-7B-step_min_g8_b384_math", "PursuitOfDataScience/Argonne-Qwen1.5-0.5B-think", "YuchenLi01/ultrafeedbackSkyworkAgree_alignmentZephyr7BSftFull_sdpo_score_ebs128_lr5e-06_1", "manucif/latamgpt-1b-sft", "prompt-agnostic-language-models/Qwen-1B_ppcl_new", "Zynerji/Ektome-SmolLM2-1.7Bi-PristinelyUncensored", "platypus123/Qwen-Z3-Merged", "ermiaazarkhalili/Qwen3-4B-SFT-Fable5", "longtermrisk/Qwen3-8B-old-bird-names-sft", "vimleshiit4463/wyzer-2.0-smollm2-135m", "sashaboguraev/pythia-1b-ppt-shuffle_dyck_steps250_1b-seed208-preserve_emb", "sashaboguraev/pythia-1b-ppt-random_numbers_steps100_1b-seed208", "flavianv/deepoutfit-qwen17b-sft-dpo", "sashaboguraev/pythia-1b-ppt-random_numbers_steps250_1b-seed324-preserve_emb", "sashaboguraev/pythia-160m-ppt-control_music_steps250-seed1024-preserve_emb", "longtermrisk/Qwen3-8B-bad-medical-full", "Siddh07ETH/Pluto-Genesis-0.6B", "tomhu/RL4TG-Qwen2.5-3B-OPD-7B-Teacher", "ayushshah/Qwen3-1.7B-UltraChat-SFT", "tomhu/RL4TG-Qwen2.5-3B-GRPO-2-Epochs", "huggingFacing/qwen2.5-7b-to-1.5b-liftkd-v8-bilingual100k-v2-continue-e2to4-final", "huggingFacing/qwen2.5-7b-to-1.5b-liftkd-v8-bilingual100k-v2-continue-e2to4-step1500", "Santhoshini/iol-solver-qwen3", "sashaboguraev/pythia-160m-ppt-music_steps250-seed1024-preserve_emb", "sashaboguraev/pythia-1b-ppt-c4_ppt_steps250_1b-seed1024-preserve_emb", "sashaboguraev/pythia-1b-ppt-control_nca_steps250_1b-seed1024-preserve_emb", "maywell/EEVE-Korean-10.8B-v1.0-16k", "sashaboguraev/pythia-160m-ppt-random_numbers_steps250-seed324", ] MTHREADS_MODELS = [ # 暂未在本轮提交，留空占位；config_content 已在 build_config_content 中就绪，未来可直接填充列表并加入 GPU_JOBS ] # 本轮仅提交 MetaX_c-500 / hygon_k100-ai / Cambricon_mlu-370-x8 这 3 张卡 # Kunlunxin_p-800（筛选结果为0，暂无可提交模型）/ Biren_166m（本轮不提交，保留代码与既有列表）/ # Mthreads_s4000（config_content 已就绪，本轮不提交）均不列入本次 GPU_JOBS GPU_JOBS: List[Tuple[str, List[str]]] = [ ("MetaX_c-500", METAX_MODELS), ("hygon_k100-ai", HYGON_MODELS), ("Cambricon_mlu-370-x8", CAMBRICON_MODELS), ] TOTAL_MODELS = sum(len(models) for _, models in GPU_JOBS) # ══════════════════════════════════════════════════════════ # 全局状态（供 /status 展示） # ══════════════════════════════════════════════════════════ _state = { "strategy_id": STRATEGY_ID, "phase": "starting", # starting | submitting | done | error "total": TOTAL_MODELS, "submitted": 0, "failed": 0, "per_account": {label: 0 for label, _, _ in ACCOUNTS}, "current_account": ACCOUNTS[0][0], "started_at": None, "finished_at": None, } _shutdown = threading.Event() # ══════════════════════════════════════════════════════════ # HTTP 服务 # ══════════════════════════════════════════════════════════ class Handler(BaseHTTPRequestHandler): def do_GET(self): if self.path == "/health": self._json({"status": "ok"}) elif self.path == "/status": self._json(_state) else: self._json({"error": "not found"}, 404) def _json(self, body: dict, code: int = 200): payload = json.dumps(body, default=str).encode() self.send_response(code) self.send_header("Content-Type", "application/json") self.send_header("Content-Length", str(len(payload))) self.end_headers() self.wfile.write(payload) def log_message(self, fmt, *args): print(f"[http] {self.address_string()} {fmt % args}", flush=True) def _run_http(): server = ThreadingHTTPServer((HTTP_HOST, HTTP_PORT), Handler) server.timeout = 1 print(f"[http] 监听 {HTTP_HOST}:{HTTP_PORT}", flush=True) while not _shutdown.is_set(): server.handle_request() server.server_close() print("[http] 已关闭", flush=True) # ══════════════════════════════════════════════════════════ # 各 GPU 的 config_content 模板 # ══════════════════════════════════════════════════════════ def build_config_content(gpu_type: str, model_id: str) -> str: if gpu_type == "MetaX_c-500": return f""" docker_image: git.modelhub.org.cn:9443/enginex-metax/vllm:0.9.1 nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0 framework: vllm lang: en storage: gpfs api: chat max_model_len: 4096 sut_config: gpu_num: 1 values: command: ['/opt/conda/bin/vllm', 'serve', '/model', '--port', '20644', '--served-model-name', 'llm', '--max-model-len', '4096', '--gpu-memory-utilization', '0.9', '--enforce-eager', '--trust-remote-code' ,'-tp', '1'] ref_config: gpu_num: 1 values: command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '4096', '--enforce-eager', '--trust-remote-code', '-tp', '1'] """ elif gpu_type == "Kunlunxin_p-800": return f""" docker_image: git.modelhub.org.cn:9443/enginex/xc-llm-kunlun nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0 framework: vllm lang: en storage: gpfs api: chat temperature: 0.4 repetition_penalty: 1.1 top_p: 0.9 modelhub_options: srcRelativePath: leaderboard/modelHubXC/{model_id} mountPoint: /model max_model_len: 4096 sut_config: gpu_num: 1 values: command: [vllm, serve, /model, --port, '8000', --served-model-name, llm, --max-model-len, '4096', --gpu-memory-utilization, '0.9', --enforce-eager, --trust-remote-code, -tp, '1'] ref_config: gpu_num: 1 values: command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len, '4096', --enforce-eager, --trust-remote-code, -tp, '1'] """ elif gpu_type == "Biren_166m": max_model_len = 4096 return f""" docker_image: git.modelhub.org.cn:9443/enginex/xc-llm-biren166m:26.01 nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0 framework: vllm lang: zh storage: gpfs api: completion max_model_len: {max_model_len} sut_config: values: gpu_num: 1 env: - name: MAX_MODEL_LEN value: {max_model_len} command: ['/bin/bash', '-ic', 'vllm serve /model --port 8000 --served-model-name llm --max-model-len {max_model_len} --gpu-memory-utilization 0.9 --enforce-eager --trust-remote-code -tp 1 --host 0.0.0.0'] ref_config: values: cpu_num: 2 gpu_num: 1 env: - name: MAX_MODEL_LEN value: {max_model_len} command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '{max_model_len}', '--enforce-eager', '--trust-remote-code', '-tp', '1'] model: llm """ elif gpu_type == "Cambricon_mlu-370-x8": return f""" docker_image: harbor.4pd.io/hardcore-tech/cambricon-mlu370-pytorch:v25.01-torch2.5.0-torchmlu1.24.1-ubuntu22.04-py310 nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0 framework: vllm storage: gpfs sut_config: values: gpu_num: 1 env: - name: MAX_MODEL_LEN value: 8192 command: ["vllm", "serve", "/model", "--port", "8000", "--served-model-name", "llm", "--max-model-len", "8192", "--trust-remote-code", "--dtype", "float16"] ref_config: values: cpu_num: 2 gpu_num: 1 env: - name: MAX_MODEL_LEN value: 8192 command: ["vllm", "serve", "/model", "--port", "80", "--served-model-name", "llm", "--max-model-len", "8192", "--trust-remote-code", "--dtype", "float16"] """ elif gpu_type == "hygon_k100-ai": return f""" docker_image: harbor.4pd.io/modelhubxc/enginex-hygon/vllm:0.9.2-patch-tokenizer nv_docker_image: harbor.4pd.io/modelhubxc/enginex-nvidia/vllm:0.11.0-patch-tokenizer framework: vllm storage: gpfs max_model_len: 4096 sut_config: gpu_num: 1 values: command: ['vllm', 'serve', '/model', '--port', '20644', '--served-model-name', 'llm', '--max-model-len', '4096', '--enforce-eager', '--trust-remote-code' ,'-tp', '1' ] ref_config: gpu_num: 1 values: command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '4096', '--enforce-eager', '--trust-remote-code', '-tp', '1'] """ elif gpu_type == "Mthreads_s4000": return f""" docker_image: git.modelhub.org.cn:9443/enginex-mthreads/vllm-musa-qy2-py310:v0.8.4-release nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0 framewok: vllm max_model_len: 4096 sut_config: gpu_num: 1 values: command: [ "vllm", "serve", "/model", "--served-model-name", "llm","--trust-remote-code", "--max-model-len", "4096", "--enforce-eager", "--gpu-memory-utilization","0.5"] ref_config: gpu_num: 1 values: command: [ "vllm","serve", "/model", "--served-model-name", "llm", "--trust-remote-code", "--max-model-len", "4096", "--enforce-eager" ] """ else: raise ValueError(f"未知的 GPU_TYPE: {gpu_type}") # ══════════════════════════════════════════════════════════ # 业务逻辑 # ══════════════════════════════════════════════════════════ def submit_task(gpu_type: str, xc_token: str, model_id: str): """返回 (code, message)；code == 0 表示提交成功。""" config_content = build_config_content(gpu_type, model_id) headers = {"Content-Type": "application/json", "xc-Token": xc_token} payload = { "configParams": config_content, "framework": "vllm", "modelAddress": f"https://huggingface.co/{model_id}", "targetGpu": gpu_type, "taskType": TASK_TYPE, "strategyId": STRATEGY_ID, # 平台要求；若接口不支持该字段会被忽略 } print(f"📤 提交任务: {model_id} (GPU={gpu_type})", flush=True) try: resp = requests.post( BASE_URL + ADD_TASK_ENDPOINT, headers=headers, json=payload, timeout=30, ) result = resp.json() print(f"status={resp.status_code} result={result}", flush=True) return result.get("code"), result.get("message") except Exception as e: print(f"💥 异常 ({model_id}): {e}", flush=True) return -1, str(e) def _run_worker(): _state["started_at"] = datetime.utcnow().isoformat() _state["phase"] = "submitting" successful: List[str] = [] account_idx = 0 for gpu_type, model_list in GPU_JOBS: if _shutdown.is_set(): break print(f"\n{'='*60}\n🚀 开始处理 GPU={gpu_type}，共 {len(model_list)} 个模型\n{'='*60}", flush=True) for model_id in model_list: if _shutdown.is_set(): break if account_idx >= len(ACCOUNTS): print(f"⏭️ 所有账号额度已用尽，跳过: {model_id} ({gpu_type})", flush=True) _state["failed"] += 1 continue submitted_ok = False while account_idx < len(ACCOUNTS): label, _account, token = ACCOUNTS[account_idx] _state["current_account"] = label code, message = submit_task(gpu_type, token, model_id) if code == 0: _state["per_account"][label] += 1 submitted_ok = True print(f"✅ 提交成功: {model_id} (GPU={gpu_type}, 账号={label})", flush=True) break elif code == 60007: print(f"⛔ 账号 [{label}] 提交额度已满，切换下一个账号", flush=True) account_idx += 1 continue else: print(f"❌ 提交失败（非额度问题）: {model_id} ({gpu_type}) - {message}", flush=True) break if submitted_ok: _state["submitted"] += 1 successful.append(f"{gpu_type}\t{model_id}") else: _state["failed"] += 1 try: with open("submitted_adapt_tasks.txt", "w", encoding="utf-8") as f: for line in successful: f.write(line + "\n") except Exception: pass _state["finished_at"] = datetime.utcnow().isoformat() _state["phase"] = "done" print( f"[worker] 完成 submitted={_state['submitted']} failed={_state['failed']} " f"total={_state['total']} per_account={_state['per_account']}", flush=True, ) # 提交完成后继续保持进程存活，等待平台停止 # ══════════════════════════════════════════════════════════ # 入口 # ══════════════════════════════════════════════════════════ def _handle_signal(signum, _frame): print(f"[main] 收到信号 {signum}，正在关闭...", flush=True) _shutdown.set() def main(): signal.signal(signal.SIGTERM, _handle_signal) signal.signal(signal.SIGINT, _handle_signal) http_thread = threading.Thread(target=_run_http, daemon=False) http_thread.start() worker_thread = threading.Thread(target=_run_worker, daemon=True) worker_thread.start() _shutdown.wait() print("[main] 等待 HTTP 服务关闭...", flush=True) http_thread.join(timeout=5) print("[main] 退出", flush=True) if __name__ == "__main__": main()