1 Commits
v2.3.0 ... main

Author SHA1 Message Date
78f8b5ad82 requeue downloads rejected for platform concurrency instead of discarding them
v2.3.0 burned through all 498 models in minutes with download_failed=498: the
platform already had 8 downloads running, every create returned "用户最多同时运行
8 个下载任务", and the worker treated a create failure as terminal (CREATE_FAILED)
and moved on.

Now a concurrency rejection puts the model back at the head of the queue and
waits for the next poll, so only genuine per-model failures (uniqueness check,
etc.) are recorded as failed.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-18 16:01:42 +08:00

21
main.py
View File

@@ -115,6 +115,9 @@ TASK_PAGE_ENDPOINT = "/api/adapt/task/page"
QUOTA_FULL_MSG = "当前等待中或运行中的异步模型验证任务数量已达上限"
_hygon_quota_full = False # zhoukaile 额度是否已打满
_bi150_quota_full = False # bi150 兜底账号额度是否已打满
# 平台下载并发上限提示(错误码 60005命中时把模型放回队列重试而不是丢弃
DOWNLOAD_BUSY_MSG = "个下载任务"
_download_busy = False
GPU_TYPE_HYGON = "hygon_k100-ai"
GPU_TYPE_BI150 = "Iluvatar_bi-150"
@@ -2645,7 +2648,14 @@ def create_download_task(token: str, model_id: str) -> bool:
print(f"✅ 下载任务已提交: {model_id}", flush=True)
return True
else:
print(f"⚠️ 下载任务业务失败 ({model_id}): {data.get('message')}", flush=True)
msg = str(data.get("message") or "")
if DOWNLOAD_BUSY_MSG in msg:
# 平台侧下载并发已满,不是这个模型的问题,稍后重试
global _download_busy
_download_busy = True
print(f"⏸️ 下载并发已满,稍后重试 ({model_id})", flush=True)
else:
print(f"⚠️ 下载任务业务失败 ({model_id}): {msg}", flush=True)
return False
else:
print(f"❌ HTTP 错误 ({model_id}): {resp.status_code} - {resp.text}", flush=True)
@@ -2929,13 +2939,20 @@ def _download_phase(token, pending_models, active_models, completed_results) ->
_state["download_failed"] += 1
# 2. 补充新任务(最多补到 MAX_CONCURRENT_DOWNLOADS 个)
global _download_busy
_download_busy = False
while len(active_models) < MAX_CONCURRENT_DOWNLOADS and pending_models and not _shutdown.is_set():
next_model = pending_models.pop(0)
if create_download_task(token, next_model):
active_models.add(next_model)
print(f"▶️ 启动下载: {next_model} (当前活跃: {len(active_models)})", flush=True)
elif _download_busy:
# 平台下载并发已满:放回队首等下一轮重试,不能当成失败丢弃,
# 否则整个列表会在几分钟内被空转消耗光v2.3.0 曾因此 498 个全部“失败”)
pending_models.insert(0, next_model)
break
else:
# 创建失败也视为完成避免卡住
# 模型本身的问题(如唯一性检查不通过),视为完成避免卡住
completed_results[next_model] = "CREATE_FAILED"
_state["download_failed"] += 1
print(f"❌ 创建失败: {next_model}", flush=True)