fix: recover eligible model GPU routes

This commit is contained in:
CoolBoy
2026-09-20 10:47:22 +08:00
parent 3077b169c5
commit 75c034d588
13 changed files with 436 additions and 29 deletions

View File

@@ -9,7 +9,7 @@ from collections import deque
from pathlib import Path
from typing import Any, Callable
from common import read_json, read_jsonl, utc_now, write_json
from common import parse_datetime, read_json, read_jsonl, utc_now, write_json
from daily_runner import DEFAULT_DAILY_RUNS_DIR, log, run_daily_batches
from gpu_strategy import DEFAULT_GPU_STRATEGY_PATH
from hf_discovery import HuggingFaceDiscovery
@@ -445,6 +445,7 @@ def _advance_architecture_history_backfill(
batch_tasks: list[dict[str, Any]] = []
page_calls = 0
page_errors = 0
last_error = ""
for _ in range(pages_per_batch):
incomplete = [
@@ -468,7 +469,9 @@ def _advance_architecture_history_backfill(
current=current_page,
page_size=page_size,
only_mine=True,
end_time=str(progress.get("cutoffAt") or now.isoformat()),
# Pass a datetime so ModelHubClient formats the platform's
# YYYY-MM-DD HH:MM:SS query instead of an ISO timestamp.
end_time=parse_datetime(progress.get("cutoffAt")) or now,
)
data = payload.get("data") or {}
records = [item for item in (data.get("records") or []) if isinstance(item, dict)]
@@ -500,6 +503,7 @@ def _advance_architecture_history_backfill(
page_errors += 1
state["listingErrors"] = int(state.get("listingErrors") or 0) + 1
state["lastError"] = f"{type(exc).__name__}: {exc}"[:500]
last_error = state["lastError"].replace("\n", " ")
break
batch_summary: dict[str, Any] = {
@@ -558,7 +562,8 @@ def _advance_architecture_history_backfill(
f"failure_logs={int(batch_summary.get('enrichmentAttempts') or 0)} "
f"accounts={completed_accounts}/{len(clients)} "
f"unique_total={int(progress.get('uniqueRecords') or 0)} "
"retention=decision_state_only"
f"retention=decision_state_only"
f"{f' error={last_error}' if last_error else ''}"
)
return progress