submit 24 models on zhoushasha: ppu_zw_810e(18)/hygon_k100-ai(4)/MetaX_c-500(1)/Biren_166m(1)

Candidate pool is the 24711 already-downloaded models, which satisfies
mechanism A's precondition that the model be present in platform storage.

Strict and relaxed condition-2 give identical results this round: of the 24007
pooled models with verify records, 24006 already have 已验证 on some card, so
relaxing to include 验证中 adds just one. The binding constraint is now how few
models lack a record on each card - Kunlunxin_p-800, Cambricon_mlu-370-x8 and
Iluvatar_bi-150 are all at zero and are left out of GPU_JOBS.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

main.py

@@ -1,10 +1,19 @@
 """
 xc_validation_strategy — 主入口
 
-启动后针对 4 张 GPU 卡（Biren_166m / Cambricon_mlu-370-x8 / MetaX_c-500 /
-Kunlunxin_p-800）分别批量提交各自筛选出的模型验证任务（/adminApi/async/task/create-contest-task，
+启动后针对 GPU_JOBS 中配置的 GPU 卡分别批量提交各自筛选出的模型验证任务
+（当前仅提交 ppu_zw_810e，其余 4 张卡 Biren_166m/Cambricon_mlu-370-x8/MetaX_c-500/
+Kunlunxin_p-800 的 config_content 模板和模型列表仍保留在代码中，未列入本次 GPU_JOBS）
+（/adminApi/async/task/create-contest-task，
 Bearer Token 认证），之后保持 HTTP 服务存活。
-同时暴露 /health（K8s 探活）和 /status（运行状态）。
+
+账号额度自动重试：如果某个模型提交时命中"当前等待中或运行中的异步模型验证
+任务数量已达上限"（账号额度已满），不算永久失败，会被留到下一轮；额度耗尽后
+本进程会原地等待 30 分钟，再自动重试所有因额度问题未提交成功的模型，如此循环，
+直至全部提交成功或进程被平台关闭——不需要重新部署新策略，循环逻辑在本进程内完成。
+非额度原因的失败（如模型已在验证中等）不会重试。
+
+同时暴露 /health（K8s 探活）和 /status（运行状态，含当前轮次/待重试数/下次重试时间）。
 """
 
 import json
@@ -24,7 +33,7 @@ BASE_URL           = os.environ.get("BASE_URL", "https://modelhub.org.cn")
 SUBMIT_ENDPOINT    = "/adminApi/async/task/create-contest-task"
 
 # 通过 curl -X POST https://modelhub.org.cn/adminApi/user/login 获取后填入
-AUTH_TOKEN        = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyQWNjb3VudCI6Inpob3VzaGFzaGEiLCJpZCI6MTQsInVzZXJSb2xlIjoibGVhZGVyYm9hcmQiLCJleHAiOjE3ODU3NDY3NTMsImlhdCI6MTc4NTE0MTk1M30.KwUuefNAFSNwq3_Pnaw2nef8ZC6WgsECQ_LMeQnKk2c"
+AUTH_TOKEN        = "eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJ1c2VyQWNjb3VudCI6Inpob3VzaGFzaGEiLCJpZCI6MTQsInVzZXJSb2xlIjoibGVhZGVyYm9hcmQiLCJleHAiOjE3OTAwNjkxMjAsImlhdCI6MTc4OTQ2NDMyMH0.KzJac6ddaZdtLvjD6ZnoK1PNEKFoXdyDn9Hh4FxU9ic"
 CONTEST_API_TOKEN = "ef1ef82f3c9efee413d602345fbe224d"
 CONTRIBUTORS      = "zhoushasha"
 TASK_TYPE         = "text-generation"
@@ -37,124 +46,61 @@ HTTP_PORT = 8080
 #  各 GPU 的模型列表（来自 filter_verified_models 脚本的筛选结果）
 # ══════════════════════════════════════════════════════════
 BIREN_MODELS = [
-    "BigRatz/LOL-AI-2026",
-    "aimeri/spoomplesmaxx-cardmaker-v1",
-    "Alibaba-DT/Logics-STEM-8B-SFT",
-    "Muneebmn123/insurance-voice-qwen25-1_5b",
-    "AnkitBirGurung/NEMO-12B-SFT-Further",
-    "danilarudenko/editorai-mini",
-    "EphemeralYou/Prompt-Refine-MiniCPM5-1B",
-    "mtepe01/mentorx-mistral-7b-automata-merged",
-    "DarkArtsForge/Helix-SCE-12B-jh",
-    "rpant/iolai26-solve",
-    "NithinAI12/NithinX-Omni-LLM-v1",
-    "ConvexAI/Luminex-34B-v0.2",
-    "codellama/CodeLlama-34b-hf",
-    "Lipas007/iol-ai-2026-qwen14b-awq",
-    "D-Z-W/finetuned-teacher",
-    "huan1999/ziya-llama-13b-medical-merged",
-    "codellama/CodeLlama-34b-Python-hf",
-    "ld4ad/gemma-2-9b-dunhuang",
-    "harindhar10/Olmo-7b_1M_Smiles_lora",
-    "allenai/Olmo-3-7B-Think-DPO",
-    "allenai/Olmo-3-32B-Think-DPO",
-    "RedHatAI/gemma-2-9b-it",
-    "prashanthsura/gemma-2-2b-legal-financial-sft",
-    "pfnet/plamo-2-8b",
-    "sail/Sailor2-20B-128K-SFT",
-    "facebook/layerskip-llama3.2-1B",
-    "Qwen/Qwen2.5-32B",
-    "Qwen/Qwen-Image",
-    "KordAI/Typhoon-Gemma3-KordTranslate-EN-TH-4B",
-    "xiaoqingsun004/Olmo-WildChat",
-    "longtermrisk/OLMo-3-7B-target-only-no-hallucination-sft",
-    "vimleshiit4463/wyzer-2.0-smollm2-135m",
-    "trl-lib/pythia-1b-deduped-tldr-sft",
+    "zipaltrivedi/dotnet-coder-14b",
 ]
 
 CAMBRICON_MODELS = [
-    "clear-blue-sky/evolai-reborn-tfm-008",
-    "clear-blue-sky/evolai-reborn-tfm-010",
-    "BigRatz/LOL-AI-2026",
-    "clear-blue-sky/evolai-reborn-tfm-001",
-    "clear-blue-sky/evolai-reborn-tfm-019",
-    "clear-blue-sky/evolai-reborn-tfm-007",
-    "aimeri/spoomplesmaxx-cardmaker-v1",
-    "aipatseer/inst_ft_qwen_0.6b_summ",
-    "clear-blue-sky/evolai-reborn-tfm-003",
-    "clear-blue-sky/evolai-reborn-tfm-004",
-    "Alibaba-DT/Logics-STEM-8B-SFT",
-    "clear-blue-sky/evolai-reborn-tfm-002",
-    "prism-ml/Ternary-Bonsai-4B-unpacked",
-    "Muneebmn123/insurance-voice-qwen25-1_5b",
-    "AnkitBirGurung/NEMO-12B-SFT-Further",
-    "danilarudenko/editorai-mini",
-    "rpant/iolai26-solve",
 ]
 
 METAX_MODELS = [
-    "Phoenix9781/evolai-tf-model-105",
-    "clear-blue-sky/evolai-reborn-tfm-008",
-    "clear-blue-sky/evolai-reborn-tfm-010",
-    "BigRatz/LOL-AI-2026",
-    "clear-blue-sky/evolai-reborn-tfm-001",
-    "clear-blue-sky/evolai-reborn-tfm-019",
-    "clear-blue-sky/evolai-reborn-tfm-007",
-    "clear-blue-sky/evolai-reborn-tfm-005",
-    "Lin2es/evolai-tfm-03o",
-    "clear-blue-sky/evolai-reborn-tfm-009",
-    "aimeri/spoomplesmaxx-cardmaker-v1",
-    "aipatseer/inst_ft_qwen_0.6b_summ",
-    "reaperdoesntknow/DualMind-TKD-Agentic-1.7B",
-    "clear-blue-sky/evolai-reborn-tfm-011",
-    "clear-blue-sky/evolai-reborn-tfm-003",
-    "clear-blue-sky/evolai-reborn-tfm-004",
-    "clear-blue-sky/evolai-reborn-tfm-002",
-    "amd/ReasonLite-0.6B",
-    "prism-ml/Ternary-Bonsai-4B-unpacked",
-    "Muneebmn123/insurance-voice-qwen25-1_5b",
-    "danilarudenko/editorai-mini",
-    "rpant/iolai26-solve",
+    "zipaltrivedi/dotnet-coder-14b",
+]
+
+HYGON_MODELS = [
+    "zipaltrivedi/dotnet-coder-14b",
+    "cds-jb/qwen3-14b-butterfly-subliminal-fullft",
+    "lllqaq/Qwen2.5-Coder-14B-Instruct-num11-v1-v2-v3-pairs-v3-triples-post-r2egym",
+    "deepmako/Mako-32B-Conductor",
 ]
 
 KUNLUNXIN_MODELS = [
-    "Patriae/patriae-cuban-dialect-model",
-    "Phoenix9781/evolai-tf-model-105",
-    "clear-blue-sky/evolai-reborn-tfm-008",
-    "clear-blue-sky/evolai-reborn-tfm-010",
-    "clear-blue-sky/evolai-reborn-tfm-001",
-    "clear-blue-sky/evolai-reborn-tfm-019",
-    "clear-blue-sky/evolai-reborn-tfm-007",
-    "clear-blue-sky/evolai-reborn-tfm-005",
-    "Lin2es/evolai-tfm-03o",
-    "clear-blue-sky/evolai-reborn-tfm-009",
-    "aimeri/spoomplesmaxx-cardmaker-v1",
-    "aipatseer/inst_ft_qwen_0.6b_summ",
-    "clear-blue-sky/evolai-reborn-tfm-011",
-    "clear-blue-sky/evolai-reborn-tfm-003",
-    "clear-blue-sky/evolai-reborn-tfm-004",
-    "clear-blue-sky/evolai-reborn-tfm-002",
-    "amd/ReasonLite-0.6B",
-    "prism-ml/Ternary-Bonsai-4B-unpacked",
-    "EthanGao123/CellHermes-v1.0",
-    "RecursiveMAS/Mixture-Science-BioMistral-7B",
-    "chenyitian-shanshu/SIRL-Gurobi",
-    "RedHatAI/gemma-2-9b-it",
-    "gaunernst/gemma-3-27b-it-qat-autoawq",
-    "promotion/qwen3-8b-simpo-avg-b2p5-g1p0-s42",
-    "llamaindex/vdr-2b-multi-v1",
-    "KordAI/Typhoon-Gemma3-KordTranslate-EN-TH-4B",
-    "TheDrummer/UnslopNemo-12B-v3",
-    "gradients-io-tournaments/tournament-tourn_c5d86c82ce819a79_20260706-b78a01d4-0a6a-49e1-9190-5e88ae329937-5DS6XMVr",
-    "ArliAI/Mistral-Nemo-12B-ArliAI-RPMax-v1.1",
 ]
 
-# 按顺序处理：Biren → Cambricon → MetaX → Kunlunxin
+PPU_MODELS = [
+    "fpadovani/eng-latn-100mb-after-ppt-Dp-10mb-ckpt500_seed10",
+    "fpadovani/ita-latn-10mb-after-ppt-Dp-100mb-ckpt500_seed3407",
+    "fpadovani/ita-latn-10mb-after-ppt-Dp-10mb-ckpt500_seed3407",
+    "richardr1126/spider-skeleton-wizard-coder-merged",
+    "fpadovani/dan-latn-10mb-after-ppt-shuff-dyck-10mb-ckpt500_seed3407",
+    "fpadovani/ita-latn-10mb-after-ppt-shuff-dyck-100mb-ckpt500_seed3407",
+    "omerkaragulmez/XbyK-0.1",
+    "fpadovani/eng-latn-10mb-after-ppt-Dp-10mb-ckpt500_seed10",
+    "fpadovani/eng-latn-10mb-100mb_seed10",
+    "flax-community/gpt2-medium-indonesian",
+    "RedHatAI/QwQ-32B-Preview-quantized.w8a8",
+    "fractalego/fact-checking",
+    "KoboldAI/GPT-J-6B-Adventure",
+    "EasierAI/Falcon-3-1B",
+    "theprint/mistral-7b-cthulhu",
+    "iwalton3/phoenix",
+    "renzhenzhen/internLM2-for-triples",
+    "u2mithrandir/epsi_tmall",
+]
+
+# 本轮提交：ppu_zw_810e(18) / hygon_k100-ai(4) / MetaX_c-500(1) / Biren_166m(1)，共24个。
+#
+# 候选池是 api_verify_model_download_status_a.txt 里那 24711 个【已下载】的模型，
+# 正好满足机制A「模型必须已下载到平台存储」的前提（v1.0.37 那次157个失败就是因为没下载）。
+#
+# 本轮严格口径与放宽口径（别的卡「已验证」vs「已验证或验证中」）结果完全相同：
+# 这批已下载模型里 24007 个有验证记录的，24006 个都已至少一张卡「已验证」，
+# 放宽只多捞出 1 个；真正的瓶颈是各卡「无记录」的模型太少——
+# Kunlunxin_p-800 / Cambricon_mlu-370-x8 / Iluvatar_bi-150 均为 0，故不列入 GPU_JOBS。
 GPU_JOBS: List[Tuple[str, List[str]]] = [
-    ("Biren_166m",           BIREN_MODELS),
-    ("Cambricon_mlu-370-x8", CAMBRICON_MODELS),
+    ("ppu_zw_810e",          PPU_MODELS),
+    ("hygon_k100-ai",        HYGON_MODELS),
     ("MetaX_c-500",          METAX_MODELS),
-    ("Kunlunxin_p-800",      KUNLUNXIN_MODELS),
+    ("Biren_166m",           BIREN_MODELS),
 ]
 TOTAL_MODELS = sum(len(models) for _, models in GPU_JOBS)
 
@@ -163,13 +109,16 @@ TOTAL_MODELS = sum(len(models) for _, models in GPU_JOBS)
 # ══════════════════════════════════════════════════════════
 _state = {
     "strategy_id":   STRATEGY_ID,
-    "phase":         "starting",   # starting | submitting | done | error
+    "phase":         "starting",   # starting | submitting | waiting_retry | done | error
     "total":         TOTAL_MODELS,
     "submitted":     0,
     "failed":        0,
     "per_gpu":       {gpu: 0 for gpu, _ in GPU_JOBS},
     "started_at":    None,
     "finished_at":   None,
+    "round":               0,     # 当前是第几轮提交
+    "quota_blocked_remaining": 0, # 因额度上限暂未提交成功、等待下一轮重试的模型数
+    "next_retry_at":       None,  # 下一轮重试的预计时间（额度耗尽等待期间）
 }
 _shutdown = threading.Event()
 
@@ -306,6 +255,96 @@ ref_config:
   values:
     command: [vllm, serve, /model, --port, '80', --served-model-name, llm, --max-model-len, '4096', --enforce-eager, --trust-remote-code, -tp, '1']
 """
+    elif gpu_type == "hygon_k100-ai":
+        return f"""
+docker_image: harbor.4pd.io/modelhubxc/enginex-hygon/vllm:0.9.2-patch-tokenizer
+nv_docker_image: harbor.4pd.io/modelhubxc/enginex-nvidia/vllm:0.11.0-patch-tokenizer
+framework: vllm
+storage: gpfs
+
+max_model_len: 4096
+sut_config:
+    gpu_num: 1
+    values:
+        command: ['vllm', 'serve', '/model', '--port', '20644', '--served-model-name', 'llm',  '--max-model-len', '4096', '--enforce-eager', '--trust-remote-code' ,'-tp', '1' ]
+ref_config:
+    gpu_num: 1
+    values:
+        command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '4096', '--enforce-eager', '--trust-remote-code', '-tp', '1']
+"""
+    elif gpu_type == "ppu_zw_810e":
+        return f"""gpu_type: ppu_zw_810e
+framework: vllm
+docker_image: harbor.4pd.io/hardcore-tech/asllm:1.10.1-pytorch2.10.0-ubuntu24.04-sail2.1.0-cuda13.0-sglang0.5.10-vllm0.19.0-py312
+nv_docker_image: harbor-contest.4pd.io/sunruoxi/vllm-openai-fix-tokenizer:v0.11.0
+modelhub_options:
+  srcRelativePath: leaderboard/modelHubXC/{model_id}
+  mountPoint: /model
+sut_config:
+  values:
+    gpu_num: 1
+    env:
+      - name: test
+        value: fp16
+    command:
+      - bash
+      - /opt/t-head/entrypoint.sh
+      - python3
+      - -m
+      - asllm.entrypoints.api_server
+      - --model
+      - /model
+      - --port
+      - '30000'
+      - --host
+      - 0.0.0.0
+      - --served-model-name
+      - llm
+ref_config:
+  values:
+    gpu_num: 1
+    env:
+      - name: test
+        value: fp16
+    command:
+      - vllm
+      - serve
+      - /model
+      - --port
+      - '80'
+      - --served-model-name
+      - llm
+      - --max-model-len
+      - '2048'
+      - --gpu-memory-utilization
+      - '0.9'
+      - --enforce-eager
+      - --trust-remote-code
+      - -tp
+      - '1'
+"""
+    
+    elif gpu_type == "Iluvatar_bi-150":
+            return f"""docker_image: harbor-contest.4pd.io/luopingyi/enginex-iluvatar-bi150/vllm:0.8.3
+nv_docker_image: harbor.4pd.io/dooke/vllm/vllm/vllm-openai:v0.11.0
+framework: vllm
+api: completion
+temperature: 0.7
+repetition_penalty: 1.2
+top_p: 0.9
+
+max_model_len: 4096
+max_tokens: 1024
+sut_config:
+  gpu_num: 1
+  values:
+    command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '4096', '--gpu-memory-utilization', '0.9', '--enforce-eager', '--trust-remote-code' ,'-tp', '1']
+ref_config:
+  gpu_num: 1
+  values:
+    command: ['vllm', 'serve', '/model', '--port', '80', '--served-model-name', 'llm', '--max-model-len', '4096', '--enforce-eager', '--trust-remote-code', '-tp', '1']  
+"""
+    
     else:
         raise ValueError(f"未知的 GPU_TYPE: {gpu_type}")
     
@@ -316,7 +355,14 @@ ref_config:
 # ══════════════════════════════════════════════════════════
 #  业务逻辑
 # ══════════════════════════════════════════════════════════
-def _submit_task(token: str, gpu_type: str, model_id: str) -> Tuple[bool, str]:
+# 账号"等待中/运行中"任务数已达上限时平台返回的业务错误信息（子串匹配）；
+# 命中这个的模型不算永久失败，会在额度腾出空位后自动重试，不会被记作 failed
+QUOTA_FULL_MSG = "当前等待中或运行中的异步模型验证任务数量已达上限"
+# 额度耗尽后，隔多久自动重试一次剩余（因额度问题未提交成功）的模型
+RETRY_INTERVAL_SECONDS = 30 * 60  # 30 分钟
+
+
+def _submit_task(token: str, gpu_type: str, model_id: str) -> Tuple[bool, str, str]:
     headers = {
         "Content-Type": "application/json",
         "Authorization": f"Bearer {token}",
@@ -349,13 +395,14 @@ def _submit_task(token: str, gpu_type: str, model_id: str) -> Tuple[bool, str]:
         if result.get("code") == 0:
             task_id = result.get("data", {}).get("id", "")
             print(f"[worker] OK  {model_id} (GPU={gpu_type})  task_id={task_id}", flush=True)
-            return True, task_id
+            return True, task_id, ""
         else:
-            print(f"[worker] FAIL {model_id} (GPU={gpu_type}): {result.get('message')}", flush=True)
-            return False, ""
+            message = result.get("message") or ""
+            print(f"[worker] FAIL {model_id} (GPU={gpu_type}): {message}", flush=True)
+            return False, "", message
     except Exception as e:
         print(f"[worker] ERROR {model_id} (GPU={gpu_type}): {e}", flush=True)
-        return False, ""
+        return False, "", str(e)
 
 
 def _run_worker():
@@ -366,35 +413,69 @@ def _run_worker():
     token = AUTH_TOKEN
     print("[worker] 使用预设 Token，跳过登录", flush=True)
 
-    for gpu_type, model_list in GPU_JOBS:
-        if _shutdown.is_set():
-            break
-        print(f"\n{'='*60}\n🚀 开始处理 GPU={gpu_type}，共 {len(model_list)} 个模型\n{'='*60}", flush=True)
+    # 待提交队列：保持 GPU_JOBS 里原有的 (gpu_type, model_id) 顺序
+    pending: List[Tuple[str, str]] = [
+        (gpu_type, model_id)
+        for gpu_type, model_list in GPU_JOBS
+        for model_id in model_list
+    ]
 
-        for model_id in model_list:
+    round_num = 0
+    while pending and not _shutdown.is_set():
+        round_num += 1
+        _state["round"] = round_num
+        _state["phase"] = "submitting"
+        _state["next_retry_at"] = None
+        print(
+            f"\n{'='*60}\n🚀 第 {round_num} 轮，待提交 {len(pending)} 个模型\n{'='*60}",
+            flush=True,
+        )
+
+        quota_blocked: List[Tuple[str, str]] = []
+        for gpu_type, model_id in pending:
             if _shutdown.is_set():
                 break
-            ok, task_id = _submit_task(token, gpu_type, model_id)
+            ok, task_id, message = _submit_task(token, gpu_type, model_id)
             if ok:
                 _state["submitted"] += 1
                 _state["per_gpu"][gpu_type] += 1
                 successful.append((task_id, gpu_type, model_id))
+            elif QUOTA_FULL_MSG in message:
+                # 账号额度暂时满了，不算永久失败，留到下一轮重试
+                quota_blocked.append((gpu_type, model_id))
             else:
+                # 非额度原因失败（如重复提交等），不再重试
                 _state["failed"] += 1
 
-    # 写入结果文件
-    try:
-        with open("submitted_validation_tasks.txt", "w", encoding="utf-8") as f:
-            for tid, gpu, mid in successful:
-                f.write(f"{tid}\t{gpu}\t{mid}\n")
-    except Exception:
-        pass
+        pending = quota_blocked
+        _state["quota_blocked_remaining"] = len(pending)
+
+        # 每轮结束都把已成功的结果落盘一次，避免中途重启丢失记录
+        try:
+            with open("submitted_validation_tasks.txt", "w", encoding="utf-8") as f:
+                for tid, gpu, mid in successful:
+                    f.write(f"{tid}\t{gpu}\t{mid}\n")
+        except Exception:
+            pass
+
+        if pending and not _shutdown.is_set():
+            next_retry = datetime.utcnow().timestamp() + RETRY_INTERVAL_SECONDS
+            _state["next_retry_at"] = datetime.utcfromtimestamp(next_retry).isoformat()
+            _state["phase"] = "waiting_retry"
+            print(
+                f"[worker] 第 {round_num} 轮结束：{len(pending)} 个模型因账号额度上限暂未提交，"
+                f"{RETRY_INTERVAL_SECONDS // 60} 分钟后自动重试（不部署新策略，本进程内循环）...",
+                flush=True,
+            )
+            _shutdown.wait(RETRY_INTERVAL_SECONDS)
 
     _state["finished_at"] = datetime.utcnow().isoformat()
     _state["phase"] = "done"
+    _state["quota_blocked_remaining"] = len(pending)
     print(
         f"[worker] 完成  submitted={_state['submitted']}  failed={_state['failed']}  "
-        f"total={_state['total']}  per_gpu={_state['per_gpu']}",
+        f"total={_state['total']}  per_gpu={_state['per_gpu']}  "
+        f"仍因额度未提交(如遇shutdown中断)={len(pending)}",
         flush=True,
     )
     # 提交完成后继续保持进程存活，等待平台停止