requeue downloads rejected for platform concurrency instead of discarding them

v2.3.0 burned through all 498 models in minutes with download_failed=498: the
platform already had 8 downloads running, every create returned "用户最多同时运行
8 个下载任务", and the worker treated a create failure as terminal (CREATE_FAILED)
and moved on.

Now a concurrency rejection puts the model back at the head of the queue and
waits for the next poll, so only genuine per-model failures (uniqueness check,
etc.) are recorded as failed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

main.py

@@ -41,11 +41,20 @@ HYGON_SUBMIT_ACCOUNT = "zhoukaile"
 HYGON_XC_TOKEN        = "bd7c52f3b9604ef48a14dd6174513935"
 
 # ── 本轮开关 ────────────────────────────────────────────────
+# 纯下载模式：当前 498 个模型的 hygon 和 bi150 都已经在前几轮提前占位完成，
+# 本轮只用 zhoushasha 跑下载任务，不再提交任何验证任务。
+# 开启后会【跳过整个占位阶段】，直接下载 ALL_MODEL_IDS 里的全部模型。
+# 换新模型列表时改回 False，并把下面两个 ENABLE_* 打开。
+DOWNLOAD_ONLY = True
+
 # hygon 已经用 zhoukaile 把当前模型列表（498个）从头交到最后一个了，
-# 再交只会全部撞 60028，所以本轮关掉 hygon，只补 bi150，让两张卡覆盖对齐。
+# 再交只会全部撞 60028。下次换新模型列表时把它改回 True。
-# 下次换新模型列表时把它改回 True。
 ENABLE_HYGON_SUBMIT = False
 
+# bi150 已由 34 个零积分小号 + zhangyuanxi 兜底占位完成（v2.2.0 补了 436 个）。
+# 下次换新模型列表时把它改回 True。
+ENABLE_BI150_SUBMIT = False
+
 # bi150 兜底账号：零积分小号（每个只占1个）用完之后，
 # 剩余模型的 bi150 统一用这个账号提交。
 # 用独立账号而不是 zhoukaile，避免和 hygon 抢同一个账号的额度。
@@ -106,6 +115,9 @@ TASK_PAGE_ENDPOINT = "/api/adapt/task/page"
 QUOTA_FULL_MSG     = "当前等待中或运行中的异步模型验证任务数量已达上限"
 _hygon_quota_full  = False   # zhoukaile 额度是否已打满
 _bi150_quota_full  = False   # bi150 兜底账号额度是否已打满
+# 平台下载并发上限提示（错误码 60005）；命中时把模型放回队列重试而不是丢弃
+DOWNLOAD_BUSY_MSG  = "个下载任务"
+_download_busy     = False
 GPU_TYPE_HYGON     = "hygon_k100-ai"
 GPU_TYPE_BI150     = "Iluvatar_bi-150"
 
@@ -2636,7 +2648,14 @@ def create_download_task(token: str, model_id: str) -> bool:
                 print(f"✅ 下载任务已提交: {model_id}", flush=True)
                 return True
             else:
-                print(f"⚠️ 下载任务业务失败 ({model_id}): {data.get('message')}", flush=True)
+                msg = str(data.get("message") or "")
+                if DOWNLOAD_BUSY_MSG in msg:
+                    # 平台侧下载并发已满，不是这个模型的问题，稍后重试
+                    global _download_busy
+                    _download_busy = True
+                    print(f"⏸️ 下载并发已满，稍后重试 ({model_id})", flush=True)
+                else:
+                    print(f"⚠️ 下载任务业务失败 ({model_id}): {msg}", flush=True)
                 return False
         else:
             print(f"❌ HTTP 错误 ({model_id}): {resp.status_code} - {resp.text}", flush=True)
@@ -2823,28 +2842,63 @@ def _run_worker():
     #  对策：把提交挪到下载【之前】。先占位再下载，下载队列即使被爬也没用。
     #  触发的 adminProxy 代下载很慢无所谓——我方自己的下载会先完成。
     # ══════════════════════════════════════════════════════
-    _state["phase"] = "reserving"
-    _init_bi150_queue()
-
     reserved_models = []          # 占位成功、需要下载的模型（按占位顺序）
 
-    print(
+    if DOWNLOAD_ONLY:
-        f"\n🔒 阶段一：占位（bi150 零积分待占位账号 {len(_bi150_pending)} 个，"
+        # 纯下载模式：占位已在前几轮完成，直接下载全部模型
-        f"用完后兜底账号={BI150_FALLBACK_ACCOUNT}；"
+        reserved_models = list(ALL_MODEL_IDS)
-        f"hygon={'启用' if ENABLE_HYGON_SUBMIT else '本轮关闭'}）\n",
+        print(
-        flush=True,
+            f"\n⏭️  纯下载模式（DOWNLOAD_ONLY=True）：跳过占位阶段，"
-    )
+            f"直接下载全部 {len(reserved_models)} 个模型，不提交任何验证任务\n",
+            flush=True,
+        )
+    else:
+        _state["phase"] = "reserving"
+        _init_bi150_queue()
+        print(
+            f"\n🔒 阶段一：占位（bi150 零积分待占位账号 {len(_bi150_pending)} 个，"
+            f"用完后兜底账号={BI150_FALLBACK_ACCOUNT}，"
+            f"bi150={'启用' if ENABLE_BI150_SUBMIT else '关闭'}；"
+            f"hygon={'启用' if ENABLE_HYGON_SUBMIT else '关闭'}）\n",
+            flush=True,
+        )
+        _reserve_phase(reserved_models)
+        print(
+            f"\n🔒 占位阶段完成：bi150={_state['reserved_bi150']}  "
+            f"hygon={_state['reserved_hygon']}  "
+            f"待下载模型={len(reserved_models)}\n",
+            flush=True,
+        )
+
+    # ══════════════════════════════════════════════════════
+    #  阶段二：下载（机制A，zhoushasha，8 并发）
+    #  只下载已占到位的模型——这些位置已经是我方的，下载完即可开跑验证
+    # ══════════════════════════════════════════════════════
+    _state["phase"] = "downloading"
+    pending_models = list(reserved_models)  # 尚未开始下载的模型
+    active_models: Set[str] = set()         # 当前正在下载的模型
+    completed_results = {}                  # model_id -> status
+
+    print(f"🚀 阶段二：下载 {len(pending_models)} 个模型。最大并发数: {MAX_CONCURRENT_DOWNLOADS}\n", flush=True)
+
+    _download_phase(token, pending_models, active_models, completed_results)
+
+
+def _reserve_phase(reserved_models: List[str]) -> None:
+    """阶段一：占位（机制B）。结果追加到 reserved_models。"""
     for model_id in ALL_MODEL_IDS:
         if _shutdown.is_set():
             break
-        if _bi150_quota_full and (_hygon_quota_full or not ENABLE_HYGON_SUBMIT):
+        bi_done = _bi150_quota_full or not ENABLE_BI150_SUBMIT
+        hy_done = _hygon_quota_full or not ENABLE_HYGON_SUBMIT
+        if bi_done and hy_done:
             print("🔒 bi150 与 hygon 均已无额度可用，占位阶段结束", flush=True)
             break
 
         got = False
 
         # bi150：先用还没有 bi150 任务的零积分账号，用完后走兜底账号
-        if not _bi150_quota_full:
+        if ENABLE_BI150_SUBMIT and not _bi150_quota_full:
             if submit_test_task_bi150(model_id):
                 _state["submitted"] += 1
                 _state["reserved_bi150"] += 1
@@ -2864,24 +2918,9 @@ def _run_worker():
         if got:
             reserved_models.append(model_id)
 
-    print(
-        f"\n🔒 占位阶段完成：bi150={_state['reserved_bi150']}  "
-        f"hygon={_state['reserved_hygon']}  "
-        f"待下载模型={len(reserved_models)}\n",
-        flush=True,
-    )
-
-    # ══════════════════════════════════════════════════════
-    #  阶段二：下载（机制A，zhoushasha，8 并发）
-    #  只下载已经占到位的模型——这些位置已经是我方的，下载完即可开跑验证
-    # ══════════════════════════════════════════════════════
-    _state["phase"] = "downloading"
-    pending_models = list(reserved_models)  # 尚未开始下载的模型
-    active_models: Set[str] = set()         # 当前正在下载的模型
-    completed_results = {}                  # model_id -> status
-
-    print(f"🚀 阶段二：下载 {len(pending_models)} 个已占位模型。最大并发数: {MAX_CONCURRENT_DOWNLOADS}\n", flush=True)
 
+def _download_phase(token, pending_models, active_models, completed_results) -> None:
+    """阶段二：下载（机制A，zhoushasha）。"""
     while (pending_models or active_models) and not _shutdown.is_set():
         # 1. 检查活跃任务状态
         for model_id in list(active_models):
@@ -2900,13 +2939,20 @@ def _run_worker():
                     _state["download_failed"] += 1
 
         # 2. 补充新任务（最多补到 MAX_CONCURRENT_DOWNLOADS 个）
+        global _download_busy
+        _download_busy = False
         while len(active_models) < MAX_CONCURRENT_DOWNLOADS and pending_models and not _shutdown.is_set():
             next_model = pending_models.pop(0)
             if create_download_task(token, next_model):
                 active_models.add(next_model)
                 print(f"▶️  启动下载: {next_model} (当前活跃: {len(active_models)})", flush=True)
+            elif _download_busy:
+                # 平台下载并发已满：放回队首等下一轮重试，不能当成失败丢弃，
+                # 否则整个列表会在几分钟内被空转消耗光（v2.3.0 曾因此 498 个全部“失败”）
+                pending_models.insert(0, next_model)
+                break
             else:
-                # 创建失败也视为完成（避免卡住）
+                # 模型本身的问题（如唯一性检查不通过），视为完成，避免卡住
                 completed_results[next_model] = "CREATE_FAILED"
                 _state["download_failed"] += 1
                 print(f"❌ 创建失败: {next_model}", flush=True)