fix: recover eligible model GPU routes

modelhub_submmit_api/README.md

@@ -115,14 +115,19 @@ bash run_poll.sh --dry-run
 - Strategy state is stored in `.modelhub_state/gpu_strategy.json`; a generation never recalculates
   during candidate submission.
 - Candidate discovery starts with the configured recent window, then automatically expands to
-  7 days, 30 days, and older history (up to 3,000 models) when the recent pool is exhausted.
+  7 days, 30 days, and the first 3,000 raw ModelScope rows ordered by update time. If the
+  candidate pool is still short, it rotates through up to 200 download-ranked rows per cycle
+  as an older-model fallback.
 - Model verification responses are cached across poll cycles for 15 minutes. Local model/GPU
   failures cool down after 24 hours instead of remaining permanently blocked.
 - Community deduplication is model/GPU-specific: another GPU's adaptation does not block the
-  current GPU. Every actual submission performs a fresh uncached check for its exact GPU.
+  current GPU. `未验证` with no records is eligible. `验证失败` is eligible only with a
+  different publicly vetted framework after all recorded attempts explicitly failed;
+  `已验证`, `验证中`, and ambiguous records remain blocked. Every actual submission
+  performs a fresh uncached check for its exact GPU and framework.
 - If the community lookup is unavailable, submission is deferred. A platform model-uniqueness
   rejection permanently excludes only that model/GPU combination from future local retries.
-- Each model can be submitted at most once per GPU.
+- The same framework is not retried for a model/GPU after a recorded failure.
 - Multiple ModelHub tokens are pooled and used to route submissions to the account with available async capacity.
 - Concurrent submissions reserve account slots locally, and an account-capacity race automatically falls through to another account.
 - Concurrent local processes claim model/GPU pairs in `.modelhub_state/submission_claims.jsonl`; shared ledger, history, and outcome files use process locks and atomic replacement.

modelhub_submmit_api/hf_discovery.py

@@ -86,7 +86,7 @@ class HuggingFaceDiscovery:
         self._model_last_modified_cache: OrderedDict[str, datetime | None] = OrderedDict()
         self._model_last_modified_lock = threading.Lock()
         self._model_page_cache: OrderedDict[
-            tuple[str, int, int], tuple[float, list[dict[str, Any]]]
+            tuple[str, str, int, int], tuple[float, list[dict[str, Any]]]
         ] = OrderedDict()
         self._model_page_cache_ttl = max(
             0.0,
@@ -108,6 +108,8 @@ class HuggingFaceDiscovery:
         self._unsupported_task_filters: dict[str, float] = {}
         self._model_card_cache: OrderedDict[str, dict[str, Any]] = OrderedDict()
         self._model_card_lock = threading.Lock()
+        self._popular_page_lock = threading.Lock()
+        self._popular_page_cursor = 0
 
     @staticmethod
     def _remember_lru(cache: OrderedDict, key: Any, value: Any, limit: int) -> None:
@@ -124,6 +126,8 @@ class HuggingFaceDiscovery:
         min_downloads: int,
         updated_after=None,
         read_concurrency: int = 1,
+        sort: str = "last_modified",
+        rotate_pages: bool = False,
     ) -> list[HFModelSummary]:
         if not pipeline_tags:
             return []
@@ -132,6 +136,17 @@ class HuggingFaceDiscovery:
         del read_concurrency
 
         per_tag_limit = limit if len(pipeline_tags) <= 1 else max(10, (limit + len(pipeline_tags) - 1) // len(pipeline_tags))
+        start_page = 1
+        if rotate_pages and sort == "downloads":
+            window_pages = max(1, min(60, (per_tag_limit + 49) // 50))
+            window_count = max(1, 60 // window_pages)
+            with self._popular_page_lock:
+                start_page = 1 + (self._popular_page_cursor % window_count) * window_pages
+                self._popular_page_cursor += 1
+            print(
+                f"[modelscope] popular_window=start_page:{start_page} pages:{window_pages}",
+                flush=True,
+            )
 
         for pipeline_tag in pipeline_tags:
             for model in self._query_recent_models(
@@ -139,6 +154,8 @@ class HuggingFaceDiscovery:
                 limit=per_tag_limit,
                 min_downloads=min_downloads,
                 updated_after=updated_after,
+                sort=sort,
+                start_page=start_page,
             ):
                 current = deduped.get(model.repo_id)
                 if current is None or (model.last_modified or parse_datetime("1970-01-01")) > (
@@ -156,6 +173,8 @@ class HuggingFaceDiscovery:
         limit: int,
         min_downloads: int,
         updated_after=None,
+        sort: str = "last_modified",
+        start_page: int = 1,
     ) -> list[HFModelSummary]:
         # ModelScope OpenAPI rejects values above 50 with InputParameterError.
         page_size = min(max(1, limit), 50)
@@ -163,8 +182,9 @@ class HuggingFaceDiscovery:
         task_tag = MODELSCOPE_TASK_TAGS.get(pipeline_tag, pipeline_tag)
         filter_disabled = self._unsupported_task_filters.get(task_tag, 0.0) > time.monotonic()
         models: list[HFModelSummary] = []
-        for page_number in range(1, (max_items + page_size - 1) // page_size + 1):
-            cache_key = ("*" if filter_disabled else task_tag, page_number, page_size)
+        page_count = (max_items + page_size - 1) // page_size
+        for page_number in range(start_page, min(61, start_page + page_count)):
+            cache_key = (sort, "*" if filter_disabled else task_tag, page_number, page_size)
             cached = self._model_page_cache.get(cache_key)
             if cached is not None and time.monotonic() - cached[0] < self._model_page_cache_ttl:
                 self._model_page_cache.move_to_end(cache_key)
@@ -177,7 +197,7 @@ class HuggingFaceDiscovery:
                     query = {
                         "page_number": page_number,
                         "page_size": page_size,
-                        "sort": "last_modified",
+                        "sort": sort,
                     }
                     if not filter_disabled:
                         query["filter.task"] = task_tag
@@ -201,6 +221,8 @@ class HuggingFaceDiscovery:
                             limit=limit,
                             min_downloads=min_downloads,
                             updated_after=updated_after,
+                            sort=sort,
+                            start_page=start_page,
                         )
                     print(
                         f"[modelscope] list_models_error task={task_tag} page={page_number} "

modelhub_submmit_api/main.py

@@ -10,7 +10,7 @@ from pathlib import Path
 from typing import Any
 
 from common import parse_datetime, runtime_instance_id, utc_now, write_json, write_jsonl
-from candidate_preflight import CandidatePreflightAdvisor
+from candidate_preflight import CandidatePreflightAdvisor, MODEL_LOAD_OVERHEAD
 from config_optimizer import SafeConfigOptimizer
 from gpu_strategy import DEFAULT_GPU_STRATEGY_PATH
 from hf_discovery import HuggingFaceDiscovery
@@ -375,6 +375,7 @@ def choose_candidate_for_gpu_detailed(
     preflight_advisor: CandidatePreflightAdvisor | None = None,
     require_official_config: bool = False,
     config_optimizer: SafeConfigOptimizer | None = None,
+    excluded_frameworks: set[str] | None = None,
 ) -> tuple[CandidateModel | None, str | None]:
     last_rejection_reason: str | None = None
     for task_type in task_types:
@@ -397,6 +398,14 @@ def choose_candidate_for_gpu_detailed(
                 incumbent_frameworks=compatible_frameworks,
                 inspection=inspection,
             )
+        if excluded_frameworks:
+            available_frameworks = [
+                framework for framework in compatible_frameworks
+                if framework not in excluded_frameworks
+            ]
+            if compatible_frameworks and not available_frameworks:
+                last_rejection_reason = "prior_failed_framework_no_alternative"
+            compatible_frameworks = available_frameworks
         if not compatible_frameworks:
             continue
 
@@ -560,8 +569,10 @@ def process_model_for_candidates(
         if hasattr(modelhub_client, "model_submission_precheck"):
             precheck = modelhub_client.model_submission_precheck(model.repo_id)
             processed_gpus = set(precheck.get("processedGpus") or set())
+            failed_frameworks_by_gpu = precheck.get("failedFrameworksByGpu") or {}
         else:
             processed_gpus = modelhub_client.processed_gpus_for_model(model.repo_id)
+            failed_frameworks_by_gpu = {}
     except Exception as exc:
         return (
             [],
@@ -580,8 +591,8 @@ def process_model_for_candidates(
     preferred_target_gpus = target_gpus[:MAX_ROUTES_PER_MODEL]
     fallback_target_gpus = target_gpus[MAX_ROUTES_PER_MODEL:]
 
-    def prepare_pending_routes(route_target_gpus: list[str]) -> list[tuple[str, list[str]]]:
-        pending: list[tuple[str, list[str]]] = []
+    def prepare_pending_routes(route_target_gpus: list[str]) -> list[tuple[str, list[str], set[str]]]:
+        pending: list[tuple[str, list[str], set[str]]] = []
         for target_gpu in route_target_gpus:
             if submission_exclusion_store is not None and submission_exclusion_store.is_blocked(model.repo_id, target_gpu):
                 skipped.append(
@@ -591,9 +602,45 @@ def process_model_for_candidates(
             if target_gpu in processed_gpus:
                 skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "already_processed_for_gpu"})
                 continue
+            if target_gpu in failed_frameworks_by_gpu and market_intelligence is None:
+                skipped.append({
+                    "repoId": model.repo_id,
+                    "targetGpu": target_gpu,
+                    "reason": "prior_failed_gpu_requires_public_framework_evidence",
+                })
+                continue
+            failed_frameworks = set(failed_frameworks_by_gpu.get(target_gpu) or set())
+            if (
+                failed_frameworks
+                and market_intelligence is not None
+                and hasattr(market_intelligence, "has_vetted_alternative_framework")
+                and allowed_task_types_set
+                and all(market_intelligence.has_framework_route(task_type, target_gpu) for task_type in allowed_task_types_set)
+                and not any(
+                    market_intelligence.has_vetted_alternative_framework(
+                        task_type, target_gpu, failed_frameworks
+                    )
+                    for task_type in allowed_task_types_set
+                )
+            ):
+                skipped.append({
+                    "repoId": model.repo_id,
+                    "targetGpu": target_gpu,
+                    "reason": "prior_failed_framework_no_alternative",
+                })
+                continue
             if outcome_tracker and outcome_tracker.is_model_gpu_failed(model.repo_id, target_gpu):
                 skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": "local_failure_cooldown_24h"})
                 continue
+            if preflight_advisor is not None and model.file_size:
+                memory_gib = preflight_advisor.gpu_memory_gib.get(target_gpu)
+                if memory_gib and model.file_size / (1024**3) * MODEL_LOAD_OVERHEAD > memory_gib:
+                    skipped.append({
+                        "repoId": model.repo_id,
+                        "targetGpu": target_gpu,
+                        "reason": "preflight_predicted_oom",
+                    })
+                    continue
 
             exact_task_types = list(allowed_task_types_set)
             if official_registry is not None:
@@ -632,7 +679,11 @@ def process_model_for_candidates(
                     }
                 )
                 continue
-            pending.append((target_gpu, compatible_task_types))
+            pending.append((
+                target_gpu,
+                compatible_task_types,
+                set(failed_frameworks_by_gpu.get(target_gpu) or set()),
+            ))
         return pending
 
     pending_task_types_by_gpu = prepare_pending_routes(preferred_target_gpus)
@@ -649,9 +700,9 @@ def process_model_for_candidates(
     except Exception as exc:
         return [], skipped, [{"repoId": model.repo_id, "reason": str(exc)}]
 
-    def evaluate_pending_routes(pending_routes: list[tuple[str, list[str]]]) -> int:
+    def evaluate_pending_routes(pending_routes: list[tuple[str, list[str], set[str]]]) -> int:
         before_count = len(candidates)
-        for target_gpu, task_types in pending_routes:
+        for target_gpu, task_types, excluded_frameworks in pending_routes:
             if len(candidates) - before_count >= MAX_ROUTES_PER_MODEL:
                 break
             best, preflight_reason = choose_candidate_for_gpu_detailed(
@@ -664,6 +715,7 @@ def process_model_for_candidates(
                 preflight_advisor=preflight_advisor,
                 require_official_config=official_registry is not None,
                 config_optimizer=config_optimizer,
+                excluded_frameworks=excluded_frameworks,
             )
             if best is None:
                 reason = (
@@ -677,6 +729,8 @@ def process_model_for_candidates(
                 skipped.append({"repoId": model.repo_id, "targetGpu": target_gpu, "reason": reason})
                 continue
             record = candidate_to_record(best)
+            if excluded_frameworks:
+                record["previousFailedFrameworks"] = sorted(excluded_frameworks)
             if outcome_tracker is not None:
                 record["transformersPrerequisiteSatisfied"] = (
                     outcome_tracker.has_non_transformers_failure(model.repo_id)
@@ -753,12 +807,29 @@ def build_adaptive_scan_stages(
                 "limit": max_models,
             }
         )
+    if allow_older_than_recent_window:
+        # ModelScope only exposes the first 3,000 raw rows for a sort order.
+        # Downloads ordering reaches a different older, established slice
+        # after the recent-update slice has produced too few candidates.
+        stages.append(
+            {
+                "name": "popular_history",
+                "updatedAfter": None,
+                "limit": min(max_models, 200),
+                "sort": "downloads",
+                "rotatePages": True,
+            }
+        )
 
     deduped: list[dict[str, Any]] = []
-    seen: set[tuple[str | None, int]] = set()
+    seen: set[tuple[str | None, int, str]] = set()
     for stage in stages:
         updated_after = stage["updatedAfter"]
-        signature = (updated_after.isoformat() if updated_after is not None else None, int(stage["limit"]))
+        signature = (
+            updated_after.isoformat() if updated_after is not None else None,
+            int(stage["limit"]),
+            str(stage.get("sort") or "last_modified"),
+        )
         if signature in seen:
             continue
         seen.add(signature)
@@ -877,6 +948,21 @@ def submit_candidate(
                 "candidate": candidate,
                 "reason": "already_processed_for_gpu",
             }
+        failed_frameworks = set(
+            (precheck.get("failedFrameworksByGpu") or {}).get(candidate["targetGpu"]) or set()
+        )
+        if candidate["framework"] in failed_frameworks:
+            return {
+                "outcome": "duplicate",
+                "candidate": candidate,
+                "reason": "prior_failed_framework_for_gpu",
+            }
+        if failed_frameworks and not candidate.get("frameworkEvidenceQualified"):
+            return {
+                "outcome": "framework_evidence_deferred",
+                "candidate": candidate,
+                "reason": "prior_failed_gpu_requires_public_framework_evidence",
+            }
 
     payload = {
         "modelAddress": candidate["modelAddress"],
@@ -1442,6 +1528,10 @@ def run_submission(
             "min_downloads": args.min_downloads,
             "updated_after": stage_updated_after,
         }
+        if stage.get("sort"):
+            query_kwargs["sort"] = stage["sort"]
+        if stage.get("rotatePages"):
+            query_kwargs["rotate_pages"] = True
         try:
             models = hf_discovery.list_recent_models(
                 **query_kwargs,
@@ -1501,6 +1591,7 @@ def run_submission(
         scanned_model_count += processed_count
         stage_summary = {
             "name": stage["name"],
+            "sort": stage.get("sort") or "last_modified",
             "updatedAfter": stage_updated_after.isoformat() if stage_updated_after is not None else None,
             "requestedLimit": stage_limit,
             "discoveredModels": len(models),
@@ -1648,10 +1739,20 @@ def run_submission(
                         {
                             "repoId": candidate["repoId"],
                             "targetGpu": candidate["targetGpu"],
-                            "reason": "already_validating_on_platform",
+                            "reason": result.get("reason") or "already_validating_on_platform",
                         }
                     )
                     continue
+                if result["outcome"] == "framework_evidence_deferred":
+                    batch_policy_skipped_candidates.append(candidate)
+                    skipped.append({
+                        "repoId": candidate["repoId"],
+                        "targetGpu": candidate["targetGpu"],
+                        "framework": candidate["framework"],
+                        "reason": result["reason"],
+                        "deferred": True,
+                    })
+                    continue
                 if result["outcome"] == "age_policy_deferred":
                     batch_policy_skipped_candidates.append(candidate)
                     skipped.append(

modelhub_submmit_api/market_intelligence.py

@@ -859,6 +859,20 @@ class MarketIntelligenceManager:
         )
         return any(bool(item.get("officialConfigValid", False)) for item in rows.values())
 
+    def has_vetted_alternative_framework(
+        self, task_type: str, target_gpu: str, excluded_frameworks: set[str]
+    ) -> bool:
+        rows = (
+            (((self.state or {}).get("frameworkStats") or {}).get(task_type) or {}).get(target_gpu)
+            or {}
+        )
+        return any(
+            framework not in excluded_frameworks
+            and bool(item.get("officialConfigValid", False))
+            and self._framework_evidence(task_type, target_gpu, framework)["qualified"]
+            for framework, item in rows.items()
+        )
+
     def ensure_framework_route(self, client: Any, task_type: str, target_gpu: str) -> None:
         if self.has_framework_route(task_type, target_gpu):
             return

modelhub_submmit_api/modelhub_client.py

@@ -36,9 +36,41 @@ def parse_model_submission_precheck(payload: Any) -> dict[str, Any]:
     if not isinstance(verify_result, dict):
         raise ModelHubAPIError("Community model precheck verifyResult is invalid", payload=payload)
 
+    blocked_gpus: set[str] = set()
+    failed_frameworks_by_gpu: dict[str, set[str]] = {}
+    for gpu, gpu_result in verify_result.items():
+        name = str(gpu)
+        if not isinstance(gpu_result, dict):
+            blocked_gpus.add(name)
+            continue
+        status = str(gpu_result.get("result") or "").strip()
+        records = gpu_result.get("records")
+        if status == "未验证" and isinstance(records, list) and not records:
+            continue
+        if status == "验证失败" and isinstance(records, list) and records:
+            frameworks: set[str] = set()
+            for record in records:
+                if not isinstance(record, dict):
+                    break
+                framework = str(record.get("framework") or "").strip()
+                try:
+                    failed = int(record.get("verifyResult")) < 0
+                except (TypeError, ValueError):
+                    failed = False
+                if not framework or not failed:
+                    break
+                frameworks.add(framework)
+            else:
+                failed_frameworks_by_gpu[name] = frameworks
+                continue
+        # Verified, validating and unknown states remain closed. A failed GPU
+        # opens only for a different framework with explicit failed records.
+        blocked_gpus.add(name)
+
     return {
         "isInDB": data.get("isInDB") is True,
-        "processedGpus": set(str(gpu) for gpu in verify_result),
+        "processedGpus": blocked_gpus,
+        "failedFrameworksByGpu": failed_frameworks_by_gpu,
         "verifyResult": verify_result,
         "modelInfo": data.get("modelInfo") if isinstance(data.get("modelInfo"), dict) else {},
     }
@@ -73,10 +105,7 @@ class ModelHubClient:
         return parse_model_submission_precheck(self.search_by_model_id(model_id, force_refresh=force_refresh))
 
     def is_model_processed_for_gpu(self, model_id: str, target_gpu: str) -> bool:
-        gpu_result = self.get_verify_result_map(model_id).get(target_gpu)
-        if gpu_result is None:
-            return False
-        return "result" in gpu_result or "records" in gpu_result
+        return target_gpu in self.model_submission_precheck(model_id)["processedGpus"]
 
     def get_verify_result_map(self, model_id: str) -> dict[str, Any]:
         payload = self.search_by_model_id(model_id)

modelhub_submmit_api/poll_runner.py

@@ -9,7 +9,7 @@ from collections import deque
 from pathlib import Path
 from typing import Any, Callable
 
-from common import read_json, read_jsonl, utc_now, write_json
+from common import parse_datetime, read_json, read_jsonl, utc_now, write_json
 from daily_runner import DEFAULT_DAILY_RUNS_DIR, log, run_daily_batches
 from gpu_strategy import DEFAULT_GPU_STRATEGY_PATH
 from hf_discovery import HuggingFaceDiscovery
@@ -445,6 +445,7 @@ def _advance_architecture_history_backfill(
     batch_tasks: list[dict[str, Any]] = []
     page_calls = 0
     page_errors = 0
+    last_error = ""
 
     for _ in range(pages_per_batch):
         incomplete = [
@@ -468,7 +469,9 @@ def _advance_architecture_history_backfill(
                 current=current_page,
                 page_size=page_size,
                 only_mine=True,
-                end_time=str(progress.get("cutoffAt") or now.isoformat()),
+                # Pass a datetime so ModelHubClient formats the platform's
+                # YYYY-MM-DD HH:MM:SS query instead of an ISO timestamp.
+                end_time=parse_datetime(progress.get("cutoffAt")) or now,
             )
             data = payload.get("data") or {}
             records = [item for item in (data.get("records") or []) if isinstance(item, dict)]
@@ -500,6 +503,7 @@ def _advance_architecture_history_backfill(
             page_errors += 1
             state["listingErrors"] = int(state.get("listingErrors") or 0) + 1
             state["lastError"] = f"{type(exc).__name__}: {exc}"[:500]
+            last_error = state["lastError"].replace("\n", " ")
             break
 
     batch_summary: dict[str, Any] = {
@@ -558,7 +562,8 @@ def _advance_architecture_history_backfill(
         f"failure_logs={int(batch_summary.get('enrichmentAttempts') or 0)} "
         f"accounts={completed_accounts}/{len(clients)} "
         f"unique_total={int(progress.get('uniqueRecords') or 0)} "
-        "retention=decision_state_only"
+        f"retention=decision_state_only"
+        f"{f' error={last_error}' if last_error else ''}"
     )
     return progress
 

modelhub_submmit_api/version.py

@@ -1 +1 @@
-AGENT_VERSION = "2026.09.18.1"
+AGENT_VERSION = "2026.09.20.1"