fix: retry failed model inspections and preserve rejection evidence

Do not cache transient repository-list failures as empty weight sets.
Deduplicate normalized file paths without lowering reported file sizes.
Record whitelisted OOM evidence and bounded discovery diagnostics.

modelhub_submmit_api/hf_discovery.py

@@ -110,6 +110,7 @@ class HuggingFaceDiscovery:
         self._model_card_lock = threading.Lock()
         self._popular_page_lock = threading.Lock()
         self._popular_page_cursor = 0
+        self.last_listing_diagnostics: list[dict[str, Any]] = []
 
     @staticmethod
     def _remember_lru(cache: OrderedDict, key: Any, value: Any, limit: int) -> None:
@@ -133,6 +134,7 @@ class HuggingFaceDiscovery:
             return []
 
         deduped: dict[str, HFModelSummary] = {}
+        self.last_listing_diagnostics = []
         del read_concurrency
 
         per_tag_limit = limit if len(pipeline_tags) <= 1 else max(10, (limit + len(pipeline_tags) - 1) // len(pipeline_tags))
@@ -156,6 +158,7 @@ class HuggingFaceDiscovery:
                 updated_after=updated_after,
                 sort=sort,
                 start_page=start_page,
+                diagnostics=self.last_listing_diagnostics,
             ):
                 current = deduped.get(model.repo_id)
                 if current is None or (model.last_modified or parse_datetime("1970-01-01")) > (
@@ -175,6 +178,7 @@ class HuggingFaceDiscovery:
         updated_after=None,
         sort: str = "last_modified",
         start_page: int = 1,
+        diagnostics: list[dict[str, Any]] | None = None,
     ) -> list[HFModelSummary]:
         # ModelScope OpenAPI rejects values above 50 with InputParameterError.
         page_size = min(max(1, limit), 50)
@@ -183,12 +187,23 @@ class HuggingFaceDiscovery:
         filter_disabled = self._unsupported_task_filters.get(task_tag, 0.0) > time.monotonic()
         models: list[HFModelSummary] = []
         page_count = (max_items + page_size - 1) // page_size
+        diagnostic = {
+            "pipelineTag": pipeline_tag, "sort": sort,
+            "requestedLimit": limit, "effectiveLimit": max_items,
+            "pageSize": page_size, "startPage": start_page,
+            "pagesRead": 0, "cachedPages": 0, "rawRowsScanned": 0,
+            "acceptedModels": 0, "stopReason": "page_budget",
+            "catalogExhausted": False,
+        }
+        if diagnostics is not None:
+            diagnostics.append(diagnostic)
         for page_number in range(start_page, min(61, start_page + page_count)):
             cache_key = (sort, "*" if filter_disabled else task_tag, page_number, page_size)
             cached = self._model_page_cache.get(cache_key)
             if cached is not None and time.monotonic() - cached[0] < self._model_page_cache_ttl:
                 self._model_page_cache.move_to_end(cache_key)
                 items = list(cached[1])
+                diagnostic["cachedPages"] += 1
             else:
                 elapsed = time.monotonic() - self._last_model_page_request_at
                 if self._last_model_page_request_at > 0 and elapsed < self._page_interval_seconds:
@@ -211,6 +226,7 @@ class HuggingFaceDiscovery:
                     self._last_model_page_request_at = time.monotonic()
                     if exc.status_code == 400 and not filter_disabled:
                         self._unsupported_task_filters[task_tag] = time.monotonic() + 86_400
+                        diagnostic["stopReason"] = "unsupported_filter_retry"
                         print(
                             f"[modelscope] task_filter_unsupported task={task_tag} "
                             "fallback=unfiltered ttl=86400s",
@@ -223,12 +239,15 @@ class HuggingFaceDiscovery:
                             updated_after=updated_after,
                             sort=sort,
                             start_page=start_page,
+                            diagnostics=diagnostics,
                         )
                     print(
                         f"[modelscope] list_models_error task={task_tag} page={page_number} "
                         f"partial_models={len(models)} retry_next_cycle=true error={exc}",
                         flush=True,
                     )
+                    diagnostic["stopReason"] = "http_error"
+                    diagnostic["httpStatus"] = exc.status_code
                     break
 
                 items = self._extract_models(payload)
@@ -238,7 +257,11 @@ class HuggingFaceDiscovery:
                     (time.monotonic(), list(items)),
                     self._page_cache_limit,
                 )
+            diagnostic["pagesRead"] += 1
+            diagnostic["rawRowsScanned"] += len(items)
             if not items:
+                diagnostic["stopReason"] = "empty_page"
+                diagnostic["catalogExhausted"] = True
                 break
             for item in items:
                 model = self._parse_model(
@@ -249,9 +272,13 @@ class HuggingFaceDiscovery:
                 )
                 if model is not None:
                     models.append(model)
+                    diagnostic["acceptedModels"] += 1
                 if len(models) >= max_items:
+                    diagnostic["stopReason"] = "result_limit"
                     return models
             if len(items) < page_size:
+                diagnostic["stopReason"] = "short_page"
+                diagnostic["catalogExhausted"] = True
                 break
         return models
 
@@ -412,7 +439,10 @@ class HuggingFaceDiscovery:
             )
         except HttpJsonError as exc:
             print(f"[modelscope] repo_tree_error repo={repo_id} error={exc}", flush=True)
-            payload = None
+            # A failed lookup is not an empty repository. Let the caller defer
+            # inspection and retry next cycle instead of caching missing
+            # weights as a permanent framework-incompatibility decision.
+            raise
 
         entries = self._extract_files(payload)
 
@@ -570,7 +600,7 @@ def inspect_repo_tree(repo_id: str, entries: list[dict[str, Any]]) -> ModelInspe
                 break
         if size_present:
             try:
-                file_sizes[path] = max(0, int(size_value))
+                file_sizes[path] = max(file_sizes.get(path, 0), int(size_value))
             except (TypeError, ValueError):
                 pass
         filename = PurePosixPath(path).name.lower()
@@ -584,12 +614,12 @@ def inspect_repo_tree(repo_id: str, entries: list[dict[str, Any]]) -> ModelInspe
     selected_gguf = choose_best_gguf(gguf_files)
     return ModelInspection(
         repo_id=repo_id,
-        file_paths=sorted(file_paths),
+        file_paths=sorted(set(file_paths)),
         file_sizes=file_sizes,
-        gguf_files=sorted(gguf_files),
+        gguf_files=sorted(set(gguf_files)),
         selected_gguf=PurePosixPath(selected_gguf).name if selected_gguf else None,
-        weight_files=sorted(vllm_weight_files),
-        onnx_files=sorted(onnx_files),
+        weight_files=sorted(set(vllm_weight_files)),
+        onnx_files=sorted(set(onnx_files)),
     )